| The Gaudi Framework v32r1 (f65d50dc)

ZipPythonDir.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2
3 # file ZipPythonDir.py
4 # Script to generate a zip file that can replace a directory in the python path.
5
6 import os
7 import sys
8 import zipfile
9 import logging
10 import stat
11 import time
12 import re
13 import codecs
14 from StringIO import StringIO
15
16 # Class for generic exception coming from the zipdir() function
17
18
19 class ZipdirError(RuntimeError):
20 pass
21
22
23 # Collect the changes to be applied to the zip file.
24 #
25 # @param directory: directory to be packed in the zip file
26 # @param infolist: list of ZipInfo objects already contained in the zip archive
27 #
28 # @return: tuple of (added, modified, untouched, removed) entries in the directory with respect to the zip file
29 #
30
31
32 def _zipChanges(directory, infolist):
33 # gets the dates of the files in the zip archive
34 infos = {}
35 for i in infolist:
36 fn = i.filename
37 if fn.endswith(".pyc"):
38 fn = fn[:-1]
39 infos[fn] = i.date_time
40
41 # gets the changes
42 added = []
43 modified = []
44 untouched = []
45 removed = []
46 all_files = set()
47
48 log = logging.getLogger("zipdir")
49 dirlen = len(directory) + 1
50 for root, dirs, files in os.walk(directory):
51 if "lib-dynload" in dirs:
52 # exclude the directory containing binary modules
53 dirs.remove("lib-dynload")
54 arcdir = root[dirlen:]
55 for f in files:
56 ext = os.path.splitext(f)[1]
57 if ext == ".py": # extensions that can enter the zip file
58 filename = os.path.join(arcdir, f)
59 all_files.add(filename)
60 if filename not in infos:
61 action = "A"
62 added.append(filename)
63 else:
64 filetime = time.localtime(
65 os.stat(os.path.join(directory,
66 filename))[stat.ST_MTIME])[:6]
67 if filetime > infos[filename]:
68 action = "M"
69 modified.append(filename)
70 else:
71 action = "U"
72 untouched.append(filename)
73 if action in ['U']:
74 log.debug(" %s -> %s", action, filename)
75 else:
76 log.info(" %s -> %s", action, filename)
77 # cases that can be ignored
78 elif (ext not in [".pyc", ".pyo", ".stamp", ".cmtref", ".confdb"]
79 and not f.startswith('.__afs')):
80 raise ZipdirError(
81 "Cannot add '%s' to the zip file, only '.py' are allowed."
82 % os.path.join(arcdir, f))
83 # check for removed files
84 for filename in infos:
85 if filename not in all_files:
86 removed.append(filename)
87 log.info(" %s -> %s", "R", filename)
88 return (added, modified, untouched, removed)
89
90
91 def checkEncoding(fileObj):
92 '''
93 Check that a file honors the declared encoding (default ASCII for Python 2
94 and UTF-8 for Python 3).
95
96 Raises a UnicodeDecodeError in case of decoding problems and LookupError if
97 the specified codec does not exists.
98
99 See http://www.python.org/dev/peps/pep-0263/
100 '''
101 from itertools import islice
102
103 # default encoding
104 if sys.version_info[0] <= 2:
105 enc = 'ascii'
106 else:
107 enc = 'utf-8'
108
109 # find the encoding of the file, if specified (in the first two lines)
110 enc_exp = re.compile(r"coding[:=]\s*([-\w.]+)")
111 for l in islice(fileObj, 2):
112 m = enc_exp.search(l)
113 if m:
114 enc = m.group(1)
115 break
116
117 if hasattr(fileObj, 'name'):
118 logging.getLogger('checkEncoding').debug('checking encoding %s on %s',
119 enc, fileObj.name)
120 else:
121 logging.getLogger('checkEncoding').debug(
122 'checking encoding %s on file object', enc)
123 # try to read the file with the declared encoding
124 fileObj.seek(0)
125 codecs.getreader(enc)(fileObj).read()
126
127
128 # Make a zip file out of a directory containing python modules
129 def zipdir(directory, no_pyc=False):
130 filename = os.path.realpath(directory + ".zip")
131 log = logging.getLogger("zipdir")
132 if not os.path.isdir(directory):
133 log.warning('directory %s missing, creating empty .zip file',
134 directory)
135 open(filename, "ab").close()
136 return
137 msg = "Zipping directory '%s'"
138 if no_pyc:
139 msg += " (without pre-compilation)"
140 log.info(msg, directory)
141
142 # Open the file in read an update mode
143 if os.path.exists(filename):
144 zipFile = open(filename, "r+b")
145 else:
146 # If the file does not exist, we need to create it.
147 # "append mode" ensures that, in case of two processes trying to
148 # create the file, they do not truncate each other file
149 zipFile = open(filename, "ab")
150
151 try:
152 if zipfile.is_zipfile(filename):
153 infolist = zipfile.ZipFile(filename).infolist()
154 else:
155 infolist = []
156 (added, modified, untouched, removed) = _zipChanges(
157 directory, infolist)
158 if added or modified or removed:
159 tempBuf = StringIO()
160 z = zipfile.PyZipFile(tempBuf, "w", zipfile.ZIP_DEFLATED)
161 for f in added + modified + untouched:
162 src = os.path.join(directory, f)
163 checkEncoding(open(src, 'rb'))
164 if no_pyc:
165 log.debug("adding '%s'", f)
166 z.write(src, f)
167 else:
168 # Remove the .pyc file to always force a re-compilation
169 if os.path.exists(src + 'c'):
170 log.debug("removing old .pyc for '%s'", f)
171 os.remove(src + 'c')
172 log.debug("adding '%s'", f)
173 z.writepy(src, os.path.dirname(f))
174 z.close()
175 zipFile.seek(0)
176 zipFile.write(tempBuf.getvalue())
177 zipFile.truncate()
178 log.info("File '%s' closed", filename)
179 else:
180 log.info("Nothing to do on '%s'", filename)
181 except UnicodeDecodeError as x:
182 log.error("Wrong encoding in file '%s':", src)
183 log.error(" %s", x)
184 log.error("Probably you forgot the line '# -*- coding: utf-8 -*-'")
185 sys.exit(1)
186 finally:
187 zipFile.close()
188
189
190 # Main function of the script.
191 # Parse arguments and call zipdir() for each directory passed as argument
192
193
194 def main(argv=None):
195 from optparse import OptionParser
196 parser = OptionParser(usage="%prog [options] directory1 [directory2 ...]")
197 parser.add_option(
198 "--no-pyc",
199 action="store_true",
200 help="copy the .py files without pre-compiling them")
201 parser.add_option(
202 "--quiet", action="store_true", help="do not print info messages")
203 parser.add_option(
204 "--debug",
205 action="store_true",
206 help="print debug messages (has priority over --quiet)")
207
208 if argv is None:
209 argv = sys.argv
210 opts, args = parser.parse_args(argv[1:])
211
212 if not args:
213 parser.error("Specify at least one directory to zip")
214
215 # Initialize the logging module
216 level = logging.INFO
217 if opts.quiet:
218 level = logging.WARNING
219 if opts.debug:
220 level = logging.DEBUG
221 logging.basicConfig(level=level)
222
223 # zip all the directories passed as arguments
224 for d in args:
225 zipdir(d, opts.no_pyc)
226
227
228 if __name__ == '__main__':
229 main()
ZipPythonDir.checkEncoding
def checkEncoding(fileObj)
Definition: ZipPythonDir.py:91
hivetimeline.read
def read(f, regex='.*', skipevents=0)
Definition: hivetimeline.py:22
ZipPythonDir.ZipdirError
Definition: ZipPythonDir.py:19
ZipPythonDir.main
def main(argv=None)
Definition: ZipPythonDir.py:194
ZipPythonDir._zipChanges
def _zipChanges(directory, infolist)
Definition: ZipPythonDir.py:32
ZipPythonDir.zipdir
def zipdir(directory, no_pyc=False)
Definition: ZipPythonDir.py:129