Coverage for tw2/core/command.py : 37%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
3import errno
4import re
5import operator
6import shutil
7import sys
8import os
9import stat
10import tempfile
11import subprocess
12import mimetypes
13from functools import reduce
14import six
15from six.moves import map
16from six.moves import zip
18try:
19 from hashlib import md5
20except ImportError:
21 import md5
23try:
24 from cStringIO import StringIO
25except ImportError:
26 from six import StringIO
28import pkg_resources
29from setuptools import Command
30from distutils import log
32from tw2.core import core
33from tw2.core import widgets
34from tw2.core import middleware
37def request_local_fake():
38 global _request_local, _request_id
39 if _request_local == None:
40 _request_local = {}
41 try:
42 return _request_local[_request_id]
43 except KeyError:
44 rl_data = {}
45 _request_local[_request_id] = rl_data
46 return rl_data
48core.request_local = request_local_fake
49_request_local = {}
50_request_id = 'whatever'
53class archive_tw2_resources(Command):
54 """
55 Setuptools command to copy and optionally compress all static resources
56 from a series of distributions and their dependencies into a directory
57 where they can be served by a fast web server.
59 To enable compression of CSS and JS files you will need to have installed a
60 Java Runtime Environment and YUICompressor
61 (http://www.julienlecomte.net/yuicompressor)
63 In order for resources from widget eggs to be properly collected these
64 need to have a 'tw2.widgets' 'widgets' entry-point which points
65 to a module which, when imported, instantiates all needed JS and CSS Links.
67 The result is laid out in the output directory in such a way that when
68 a web server such as Apache or Nginx is configured to map URLS that
69 begin with /resources to that directory static files will be served
70 from there bypassing python completely.
73 To integrate this command into your build process you can add these lines
74 to ``setup.cfg``::
76 [archive_tw2_resources]
77 output = /home/someuser/public_html/resources/
78 compresslevel = 2
79 distributions = MyProject
80 yuicompressor = /home/someuser/bin/yuicompressor.jar
81 onepass = true
83 [aliases]
84 deploy = archive_tw2_resources --force install
86 This way you can run::
88 $ python setup.py deploy
90 To install a new version of your app and copy/compress resources.
91 """
92 description = "Copies ToscaWidgets static resources into a directory "\
93 "where a fast web-server can serve them."
94 user_options = [
95 ("output=", "o",
96 "Output directory. If it doesn't exist it will be created."),
97 ("force", "f", "If output dir exists, it will be ovewritten"),
98 ("onepass", None, "If given, yuicompressor will only be called once "\
99 "for each kind of file with a all files "\
100 "together and then separated back into smaller "\
101 "files"),
102 ("compresslevel=", "c",
103 "Compression level: 0) for no compression (default). "\
104 "1) for js-minification. "\
105 "2) for js & css compression"),
106 ("yuicompressor=", None, "Name of the yuicompressor jar."),
107 ("distributions=", "d",
108 "List of widget dists. to include resources from "
109 "(dependencies will be handled recursively). Note that "
110 "these distributions need to define a 'tw2.widgets' "
111 "'widgets' entrypoint pointing to a a module where "
112 "resources are located."),
113 ]
115 IGNORED_NAMES = [".svn", ".git", ".hg"]
116 """
117 A list of names to ignore, used to prevent collecting
118 subversion control data.
119 """
121 def initialize_options(self):
122 self.output = ''
123 self.force = False
124 self.onepass = False
125 self.compresslevel = 0
126 self.distributions = []
127 self.yuicompressor = 'yuicompressor.jar'
129 def finalize_options(self):
130 self.ensure_string("output")
131 self.ensure_string("yuicompressor")
132 self.ensure_string_list("distributions")
133 self.compresslevel = int(self.compresslevel)
134 self.yuicompressor = os.path.abspath(self.yuicompressor)
136 def run(self):
137 if not self.output:
138 print("Need to specify an output directory", file=sys.stderr)
139 return
140 if not self.distributions:
141 print("Need to specify at least one distribution", file=sys.stderr)
142 return
143 if os.path.exists(self.output) and not self.force:
144 print((
145 "Destination dir %s exists. " % self.output) + \
146 "Use -f to overwrite.", file=sys.stderr)
147 return
148 if self.compresslevel > 0 and not os.path.exists(self.yuicompressor):
149 print("Could not find YUICompressor at " + \
150 self.yuicompressor, file=sys.stderr)
151 return
153 self.tempdir = tempdir = tempfile.mktemp()
154 self.execute(os.makedirs, (tempdir,), "Creating temp dir %s" % tempdir)
156 if self.compresslevel > 0:
157 if self.onepass:
158 self.writer = OnePassCompressingWriter(self, tempdir)
159 else:
160 self.writer = CompressingWriter(self, tempdir)
161 else:
162 self.writer = FileWriter(self, tempdir)
164 self.execute(self._copy_resources, tuple(), "Extracting resources")
165 self.writer.finalize()
166 if os.path.exists(self.output):
167 self.execute(shutil.rmtree, (self.output,),
168 "Deleting old output dir %s" % self.output)
169 self.execute(os.makedirs, (self.output,), "Creating output dir")
171 prefix = '/resources' # TODO -- get this from config.
173 final_dest = os.path.join(self.output, prefix.strip('/'))
174 self.execute(shutil.move, (tempdir, final_dest),
175 "Moving build to %s" % final_dest)
177 def _load_widgets(self, mod):
178 """ Register the widgets' resources with the middleware. """
179 print("Doing", mod.__name__)
180 for key, value in six.iteritems(mod.__dict__):
181 if isinstance(value, widgets.WidgetMeta):
182 try:
183 value(id='fake').req().prepare()
184 except Exception:
185 self.announce("Failed to register %s" % key)
187 for res in value.resources:
188 try:
189 res.req().prepare()
190 except Exception:
191 self.announce("Failed to register %s" % key)
193 def _load_widget_entry_points(self, distribution):
194 try:
195 dist = pkg_resources.get_distribution(distribution)
196 requires = [r.project_name for r in dist.requires()]
198 list(map(self._load_widget_entry_points, requires))
200 #Here we only look for a [tw2.widgets] entry point listing and we
201 #don't care what data is listed in it. We do this, because many of
202 #the existing tw2 libraries do not conform to a standard, e.g.:
203 #
204 # ## Doing it wrong:
205 # [tw2.widgets]
206 # tw2.core = tw2.core
207 #
208 # ## Doing it right:
209 # [tw2.widgets]
210 # widgets = tw2.jquery
211 #
212 #For now, anything with a [tw2.widgets] listing at all is loaded.
213 #TODO -- this should be resolved and standardized in the future.
215 for ep in pkg_resources.iter_entry_points('tw2.widgets'):
216 if ep.dist == dist:
217 mod = ep.load()
218 self._load_widgets(mod)
219 self.announce("Loaded %s" % mod.__name__)
221 except ImportError as e:
222 self.announce("%s has no widgets entrypoint" % distribution)
224 def _copy_resources(self):
226 # Set up fake middleware with which widgets can register their
227 # resources
228 core.request_local = request_local_fake
229 core.request_local()['middleware'] = middleware.make_middleware()
231 # Load widgets and have them prepare their resources
232 list(map(self._load_widget_entry_points, self.distributions))
234 rl_resources = core.request_local().setdefault('resources', [])
236 for resource in rl_resources:
237 try:
238 modname = resource.modname
239 fbase = resource.filename.split('/')[0]
240 self.execute(self._copy_resource_tree, (modname, fbase),
241 "Copying %s recursively into %s" %
242 (modname, self.writer.base))
243 except AttributeError as e:
244 pass
246 def _copy_resource_tree(self, modname, fname):
247 try:
248 for name in pkg_resources.resource_listdir(modname, fname):
249 if name in self.IGNORED_NAMES:
250 continue
251 name = '/'.join((fname, name))
252 rel_name = '/'.join((modname, name))
253 if pkg_resources.resource_isdir(modname, name):
254 self.execute(self._copy_resource_tree, (modname, name),
255 "Recursing into " + rel_name)
256 else:
257 full_name = pkg_resources.resource_filename(modname, name)
258 ct, _ = mimetypes.guess_type(full_name)
259 stream = pkg_resources.resource_stream(modname, name)
260 filename = '/'.join((modname, name))
261 self.execute(self.writer.write_file, (stream, filename),
262 "Processing " + filename)
263 stream.close()
264 except OSError as e:
265 if e.errno == errno.ENOENT:
266 self.warn("Could not copy %s" % repr((modname, fname, e)))
269class FileWriter(object):
270 def __init__(self, cmd, base):
271 self.base = base
272 self.cmd = cmd
274 def finalize(self):
275 pass
277 def write_file(self, stream, path):
278 final = os.path.join(self.base, path)
279 if not os.path.exists(os.path.dirname(final)):
280 os.makedirs(os.path.dirname(final))
281 dest = open(final, 'wb')
282 self.announce("Writing %s" % path)
283 shutil.copyfileobj(stream, dest)
284 dest.close()
286 # Delegate methods to Command
287 for name in "warn announce error execute".split():
288 exec("""\
289def %(name)s(self, *args, **kw):
290 return self.cmd.%(name)s(*args, **kw)
291""" % locals())
294class CompressingWriter(FileWriter):
296 def __init__(self, *args, **kw):
297 super(CompressingWriter, self).__init__(*args, **kw)
298 self.counters = 0, 0
300 def finalize(self):
301 try:
302 avg = reduce(operator.truediv, self.counters) * 100
303 msg = "Total JS&CSS compressed size is %.2f%% of original" % avg
304 self.announce(msg)
305 except ZeroDivisionError:
306 # No files were compressed
307 pass
309 def compress(self, stream, path):
310 typ = path.split('.')[-1]
311 if typ not in ('css', 'js'):
312 return stream
313 args = ['java', '-jar', self.cmd.yuicompressor, '--type', typ]
314 if self.cmd.compresslevel < 2:
315 args.append('--nomunge')
316 args.append('--charset=utf8')
317 p = subprocess.Popen(args, stdout=subprocess.PIPE,
318 stdin=subprocess.PIPE,
319 stderr=subprocess.PIPE)
320 self.announce("Compressing %s" % path)
321 buffer = StringIO()
322 shutil.copyfileobj(stream, buffer)
323 data = buffer.getvalue()
324 if not data:
325 return buffer
326 stdout, stderr = p.communicate(data)
327 if p.returncode != 0:
328 self.warn("Failed to compress %s: %d" % (path, p.returncode))
329 self.warn("File will be copied untouched")
330 sys.stderr.write(stderr)
331 sys.stderr.write(stdout)
332 stream.seek(0)
333 else:
334 count = len(stdout), len(data)
335 ratio = reduce(operator.truediv, count)
336 self.counters = list(map(sum, zip(self.counters, count)))
337 msg = "Compressed %s (New size: %.2f%%)" % (path, ratio * 100)
338 self.announce(msg)
339 stream = StringIO(stdout)
340 return stream
342 def write_file(self, stream, path):
343 stream = self.compress(stream, path)
344 return super(CompressingWriter, self).write_file(stream, path)
347class OnePassCompressingWriter(CompressingWriter):
348 def __init__(self, *args, **kw):
349 super(OnePassCompressingWriter, self).__init__(*args, **kw)
350 #XXX This comment trick only works with JS as of YUICompressor 2.3.5
351 self._caches = {'js': StringIO()}
352 self._marker = "/*! MARKER #### %(path)s #### MARKER */"
353 regexp = r"^\/\* MARKER #### (?P<path>.*?) #### MARKER \*\/$"
354 self._re = re.compile(regexp)
356 def _demultiplex(self, stream):
357 cur_file = None
358 buffer = StringIO()
359 stream.seek(0)
360 for line in stream:
361 m = self._re.match(line)
362 if m:
363 if cur_file:
364 buffer.seek(0)
365 FileWriter.write_file(self, buffer, cur_file)
366 buffer.truncate(0)
367 cur_file = m.group('path')
368 else:
369 buffer.write(line)
371 def finalize(self):
372 self.announce("Compressing all defered files")
373 for typ, cache in six.iteritems(self._caches):
374 cache.seek(0)
375 # self.compress only wants to know the file extension to see
376 # what kind of file it is, we pass a dummy one
377 compressed = self.compress(cache, '__defered__.' + typ)
378 self._demultiplex(compressed)
379 super(OnePassCompressingWriter, self).finalize()
381 def write_file(self, stream, path):
382 typ = path.split('.')[-1]
383 cache = self._caches.get(typ)
384 if not cache:
385 self.announce("Will not consider %s for onepass" % path)
386 return CompressingWriter.write_file(self, stream, path)
387 print(self._marker % locals(), file=cache)
388 self.announce("Defering %s for compression in one pass" % path)
389 shutil.copyfileobj(stream, cache)