Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2 

3import errno 

4import re 

5import operator 

6import shutil 

7import sys 

8import os 

9import stat 

10import tempfile 

11import subprocess 

12import mimetypes 

13from functools import reduce 

14import six 

15from six.moves import map 

16from six.moves import zip 

17 

18try: 

19 from hashlib import md5 

20except ImportError: 

21 import md5 

22 

23try: 

24 from cStringIO import StringIO 

25except ImportError: 

26 from six import StringIO 

27 

28import pkg_resources 

29from setuptools import Command 

30from distutils import log 

31 

32from tw2.core import core 

33from tw2.core import widgets 

34from tw2.core import middleware 

35 

36 

37def request_local_fake(): 

38 global _request_local, _request_id 

39 if _request_local == None: 

40 _request_local = {} 

41 try: 

42 return _request_local[_request_id] 

43 except KeyError: 

44 rl_data = {} 

45 _request_local[_request_id] = rl_data 

46 return rl_data 

47 

48core.request_local = request_local_fake 

49_request_local = {} 

50_request_id = 'whatever' 

51 

52 

53class archive_tw2_resources(Command): 

54 """ 

55 Setuptools command to copy and optionally compress all static resources 

56 from a series of distributions and their dependencies into a directory 

57 where they can be served by a fast web server. 

58 

59 To enable compression of CSS and JS files you will need to have installed a 

60 Java Runtime Environment and YUICompressor 

61 (http://www.julienlecomte.net/yuicompressor) 

62 

63 In order for resources from widget eggs to be properly collected these 

64 need to have a 'tw2.widgets' 'widgets' entry-point which points 

65 to a module which, when imported, instantiates all needed JS and CSS Links. 

66 

67 The result is laid out in the output directory in such a way that when 

68 a web server such as Apache or Nginx is configured to map URLS that 

69 begin with /resources to that directory static files will be served 

70 from there bypassing python completely. 

71 

72 

73 To integrate this command into your build process you can add these lines 

74 to ``setup.cfg``:: 

75 

76 [archive_tw2_resources] 

77 output = /home/someuser/public_html/resources/ 

78 compresslevel = 2 

79 distributions = MyProject 

80 yuicompressor = /home/someuser/bin/yuicompressor.jar 

81 onepass = true 

82 

83 [aliases] 

84 deploy = archive_tw2_resources --force install 

85 

86 This way you can run:: 

87 

88 $ python setup.py deploy 

89 

90 To install a new version of your app and copy/compress resources. 

91 """ 

92 description = "Copies ToscaWidgets static resources into a directory "\ 

93 "where a fast web-server can serve them." 

94 user_options = [ 

95 ("output=", "o", 

96 "Output directory. If it doesn't exist it will be created."), 

97 ("force", "f", "If output dir exists, it will be ovewritten"), 

98 ("onepass", None, "If given, yuicompressor will only be called once "\ 

99 "for each kind of file with a all files "\ 

100 "together and then separated back into smaller "\ 

101 "files"), 

102 ("compresslevel=", "c", 

103 "Compression level: 0) for no compression (default). "\ 

104 "1) for js-minification. "\ 

105 "2) for js & css compression"), 

106 ("yuicompressor=", None, "Name of the yuicompressor jar."), 

107 ("distributions=", "d", 

108 "List of widget dists. to include resources from " 

109 "(dependencies will be handled recursively). Note that " 

110 "these distributions need to define a 'tw2.widgets' " 

111 "'widgets' entrypoint pointing to a a module where " 

112 "resources are located."), 

113 ] 

114 

115 IGNORED_NAMES = [".svn", ".git", ".hg"] 

116 """ 

117 A list of names to ignore, used to prevent collecting 

118 subversion control data. 

119 """ 

120 

121 def initialize_options(self): 

122 self.output = '' 

123 self.force = False 

124 self.onepass = False 

125 self.compresslevel = 0 

126 self.distributions = [] 

127 self.yuicompressor = 'yuicompressor.jar' 

128 

129 def finalize_options(self): 

130 self.ensure_string("output") 

131 self.ensure_string("yuicompressor") 

132 self.ensure_string_list("distributions") 

133 self.compresslevel = int(self.compresslevel) 

134 self.yuicompressor = os.path.abspath(self.yuicompressor) 

135 

136 def run(self): 

137 if not self.output: 

138 print("Need to specify an output directory", file=sys.stderr) 

139 return 

140 if not self.distributions: 

141 print("Need to specify at least one distribution", file=sys.stderr) 

142 return 

143 if os.path.exists(self.output) and not self.force: 

144 print(( 

145 "Destination dir %s exists. " % self.output) + \ 

146 "Use -f to overwrite.", file=sys.stderr) 

147 return 

148 if self.compresslevel > 0 and not os.path.exists(self.yuicompressor): 

149 print("Could not find YUICompressor at " + \ 

150 self.yuicompressor, file=sys.stderr) 

151 return 

152 

153 self.tempdir = tempdir = tempfile.mktemp() 

154 self.execute(os.makedirs, (tempdir,), "Creating temp dir %s" % tempdir) 

155 

156 if self.compresslevel > 0: 

157 if self.onepass: 

158 self.writer = OnePassCompressingWriter(self, tempdir) 

159 else: 

160 self.writer = CompressingWriter(self, tempdir) 

161 else: 

162 self.writer = FileWriter(self, tempdir) 

163 

164 self.execute(self._copy_resources, tuple(), "Extracting resources") 

165 self.writer.finalize() 

166 if os.path.exists(self.output): 

167 self.execute(shutil.rmtree, (self.output,), 

168 "Deleting old output dir %s" % self.output) 

169 self.execute(os.makedirs, (self.output,), "Creating output dir") 

170 

171 prefix = '/resources' # TODO -- get this from config. 

172 

173 final_dest = os.path.join(self.output, prefix.strip('/')) 

174 self.execute(shutil.move, (tempdir, final_dest), 

175 "Moving build to %s" % final_dest) 

176 

177 def _load_widgets(self, mod): 

178 """ Register the widgets' resources with the middleware. """ 

179 print("Doing", mod.__name__) 

180 for key, value in six.iteritems(mod.__dict__): 

181 if isinstance(value, widgets.WidgetMeta): 

182 try: 

183 value(id='fake').req().prepare() 

184 except Exception: 

185 self.announce("Failed to register %s" % key) 

186 

187 for res in value.resources: 

188 try: 

189 res.req().prepare() 

190 except Exception: 

191 self.announce("Failed to register %s" % key) 

192 

193 def _load_widget_entry_points(self, distribution): 

194 try: 

195 dist = pkg_resources.get_distribution(distribution) 

196 requires = [r.project_name for r in dist.requires()] 

197 

198 list(map(self._load_widget_entry_points, requires)) 

199 

200 #Here we only look for a [tw2.widgets] entry point listing and we 

201 #don't care what data is listed in it. We do this, because many of 

202 #the existing tw2 libraries do not conform to a standard, e.g.: 

203 # 

204 # ## Doing it wrong: 

205 # [tw2.widgets] 

206 # tw2.core = tw2.core 

207 # 

208 # ## Doing it right: 

209 # [tw2.widgets] 

210 # widgets = tw2.jquery 

211 # 

212 #For now, anything with a [tw2.widgets] listing at all is loaded. 

213 #TODO -- this should be resolved and standardized in the future. 

214 

215 for ep in pkg_resources.iter_entry_points('tw2.widgets'): 

216 if ep.dist == dist: 

217 mod = ep.load() 

218 self._load_widgets(mod) 

219 self.announce("Loaded %s" % mod.__name__) 

220 

221 except ImportError as e: 

222 self.announce("%s has no widgets entrypoint" % distribution) 

223 

224 def _copy_resources(self): 

225 

226 # Set up fake middleware with which widgets can register their 

227 # resources 

228 core.request_local = request_local_fake 

229 core.request_local()['middleware'] = middleware.make_middleware() 

230 

231 # Load widgets and have them prepare their resources 

232 list(map(self._load_widget_entry_points, self.distributions)) 

233 

234 rl_resources = core.request_local().setdefault('resources', []) 

235 

236 for resource in rl_resources: 

237 try: 

238 modname = resource.modname 

239 fbase = resource.filename.split('/')[0] 

240 self.execute(self._copy_resource_tree, (modname, fbase), 

241 "Copying %s recursively into %s" % 

242 (modname, self.writer.base)) 

243 except AttributeError as e: 

244 pass 

245 

246 def _copy_resource_tree(self, modname, fname): 

247 try: 

248 for name in pkg_resources.resource_listdir(modname, fname): 

249 if name in self.IGNORED_NAMES: 

250 continue 

251 name = '/'.join((fname, name)) 

252 rel_name = '/'.join((modname, name)) 

253 if pkg_resources.resource_isdir(modname, name): 

254 self.execute(self._copy_resource_tree, (modname, name), 

255 "Recursing into " + rel_name) 

256 else: 

257 full_name = pkg_resources.resource_filename(modname, name) 

258 ct, _ = mimetypes.guess_type(full_name) 

259 stream = pkg_resources.resource_stream(modname, name) 

260 filename = '/'.join((modname, name)) 

261 self.execute(self.writer.write_file, (stream, filename), 

262 "Processing " + filename) 

263 stream.close() 

264 except OSError as e: 

265 if e.errno == errno.ENOENT: 

266 self.warn("Could not copy %s" % repr((modname, fname, e))) 

267 

268 

269class FileWriter(object): 

270 def __init__(self, cmd, base): 

271 self.base = base 

272 self.cmd = cmd 

273 

274 def finalize(self): 

275 pass 

276 

277 def write_file(self, stream, path): 

278 final = os.path.join(self.base, path) 

279 if not os.path.exists(os.path.dirname(final)): 

280 os.makedirs(os.path.dirname(final)) 

281 dest = open(final, 'wb') 

282 self.announce("Writing %s" % path) 

283 shutil.copyfileobj(stream, dest) 

284 dest.close() 

285 

286 # Delegate methods to Command 

287 for name in "warn announce error execute".split(): 

288 exec("""\ 

289def %(name)s(self, *args, **kw): 

290 return self.cmd.%(name)s(*args, **kw) 

291""" % locals()) 

292 

293 

294class CompressingWriter(FileWriter): 

295 

296 def __init__(self, *args, **kw): 

297 super(CompressingWriter, self).__init__(*args, **kw) 

298 self.counters = 0, 0 

299 

300 def finalize(self): 

301 try: 

302 avg = reduce(operator.truediv, self.counters) * 100 

303 msg = "Total JS&CSS compressed size is %.2f%% of original" % avg 

304 self.announce(msg) 

305 except ZeroDivisionError: 

306 # No files were compressed 

307 pass 

308 

309 def compress(self, stream, path): 

310 typ = path.split('.')[-1] 

311 if typ not in ('css', 'js'): 

312 return stream 

313 args = ['java', '-jar', self.cmd.yuicompressor, '--type', typ] 

314 if self.cmd.compresslevel < 2: 

315 args.append('--nomunge') 

316 args.append('--charset=utf8') 

317 p = subprocess.Popen(args, stdout=subprocess.PIPE, 

318 stdin=subprocess.PIPE, 

319 stderr=subprocess.PIPE) 

320 self.announce("Compressing %s" % path) 

321 buffer = StringIO() 

322 shutil.copyfileobj(stream, buffer) 

323 data = buffer.getvalue() 

324 if not data: 

325 return buffer 

326 stdout, stderr = p.communicate(data) 

327 if p.returncode != 0: 

328 self.warn("Failed to compress %s: %d" % (path, p.returncode)) 

329 self.warn("File will be copied untouched") 

330 sys.stderr.write(stderr) 

331 sys.stderr.write(stdout) 

332 stream.seek(0) 

333 else: 

334 count = len(stdout), len(data) 

335 ratio = reduce(operator.truediv, count) 

336 self.counters = list(map(sum, zip(self.counters, count))) 

337 msg = "Compressed %s (New size: %.2f%%)" % (path, ratio * 100) 

338 self.announce(msg) 

339 stream = StringIO(stdout) 

340 return stream 

341 

342 def write_file(self, stream, path): 

343 stream = self.compress(stream, path) 

344 return super(CompressingWriter, self).write_file(stream, path) 

345 

346 

347class OnePassCompressingWriter(CompressingWriter): 

348 def __init__(self, *args, **kw): 

349 super(OnePassCompressingWriter, self).__init__(*args, **kw) 

350 #XXX This comment trick only works with JS as of YUICompressor 2.3.5 

351 self._caches = {'js': StringIO()} 

352 self._marker = "/*! MARKER #### %(path)s #### MARKER */" 

353 regexp = r"^\/\* MARKER #### (?P<path>.*?) #### MARKER \*\/$" 

354 self._re = re.compile(regexp) 

355 

356 def _demultiplex(self, stream): 

357 cur_file = None 

358 buffer = StringIO() 

359 stream.seek(0) 

360 for line in stream: 

361 m = self._re.match(line) 

362 if m: 

363 if cur_file: 

364 buffer.seek(0) 

365 FileWriter.write_file(self, buffer, cur_file) 

366 buffer.truncate(0) 

367 cur_file = m.group('path') 

368 else: 

369 buffer.write(line) 

370 

371 def finalize(self): 

372 self.announce("Compressing all defered files") 

373 for typ, cache in six.iteritems(self._caches): 

374 cache.seek(0) 

375 # self.compress only wants to know the file extension to see 

376 # what kind of file it is, we pass a dummy one 

377 compressed = self.compress(cache, '__defered__.' + typ) 

378 self._demultiplex(compressed) 

379 super(OnePassCompressingWriter, self).finalize() 

380 

381 def write_file(self, stream, path): 

382 typ = path.split('.')[-1] 

383 cache = self._caches.get(typ) 

384 if not cache: 

385 self.announce("Will not consider %s for onepass" % path) 

386 return CompressingWriter.write_file(self, stream, path) 

387 print(self._marker % locals(), file=cache) 

388 self.announce("Defering %s for compression in one pass" % path) 

389 shutil.copyfileobj(stream, cache)