| # Gaudi Framework, version v25r0 |

Home |  | Generated: Mon Feb 17 2014

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
ZipPythonDir.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2
3 ## file ZipPythonDir.py
4 # Script to generate a zip file that can replace a directory in the python path.
5
6 import os
7 import sys
8 import zipfile
9 import logging
10 import stat
11 import time
12 import re
13 import codecs
14 from StringIO import StringIO
15
16 # Add to the path the entry needed to import the locker module.
17 import locker
18
19 ## Class for generic exception coming from the zipdir() function
20 class ZipdirError(RuntimeError):
21 pass
22
23 ## Collect the changes to be applied to the zip file.
24 #
25 # @param directory: directory to be packed in the zip file
26 # @param infolist: list of ZipInfo objects already contained in the zip archive
27 #
28 # @return: tuple of (added, modified, untouched, removed) entries in the directory with respect to the zip file
29 #
30 def _zipChanges(directory, infolist):
31 # gets the dates of the files in the zip archive
32 infos = {}
33 for i in infolist:
34 fn = i.filename
35 if fn.endswith(".pyc"):
36 fn = fn[:-1]
37 infos[fn] = i.date_time
38
39 # gets the changes
40 added = []
41 modified = []
42 untouched = []
43 removed = []
44 all_files = set()
45
46 log = logging.getLogger("zipdir")
47 dirlen = len(directory) + 1
48 for root, dirs, files in os.walk(directory):
49 if "lib-dynload" in dirs:
50 # exclude the directory containing binary modules
51 dirs.remove("lib-dynload")
52 arcdir = root[dirlen:]
53 for f in files:
54 ext = os.path.splitext(f)[1]
55 if ext == ".py": # extensions that can enter the zip file
56 filename = os.path.join(arcdir, f)
57 all_files.add(filename)
58 if filename not in infos:
59 action = "A"
60 added.append(filename)
61 else:
62 filetime = time.localtime(os.stat(os.path.join(directory,filename))[stat.ST_MTIME])[:6]
63 if filetime > infos[filename]:
64 action = "M"
65 modified.append(filename)
66 else:
67 action = "U"
68 untouched.append(filename)
69 if action in ['U']:
70 log.debug(" %s -> %s", action, filename)
71 else:
72 log.info(" %s -> %s", action, filename)
73 # cases that can be ignored
74 elif ext not in [".pyc", ".pyo", ".stamp", ".cmtref"] and not f.startswith('.__afs'):
75 raise ZipdirError("Cannot add '%s' to the zip file, only '.py' are allowed." % os.path.join(arcdir, f))
76 # check for removed files
77 for filename in infos:
78 if filename not in all_files:
79 removed.append(filename)
80 log.info(" %s -> %s", "R", filename)
81 return (added, modified, untouched, removed)
82
83 def checkEncoding(fileObj):
84 '''
85 Check that a file honors the declared encoding (default ASCII for Python 2
86 and UTF-8 for Python 3).
87
88 Raises a UnicodeDecodeError in case of decoding problems and LookupError if
89 the specified codec does not exists.
90
91 See http://www.python.org/dev/peps/pep-0263/
92 '''
93 from itertools import islice
94
95 # default encoding
96 if sys.version_info[0] <= 2:
97 enc = 'ascii'
98 else:
99 enc = 'utf-8'
100
101 # find the encoding of the file, if specified (in the first two lines)
102 enc_exp = re.compile(r"coding[:=]\s*([-\w.]+)")
103 for l in islice(fileObj, 2):
104 m = enc_exp.search(l)
105 if m:
106 enc = m.group(1)
107 break
108
109 if hasattr(fileObj, 'name'):
110 logging.getLogger('checkEncoding').debug('checking encoding %s on %s',
111 enc, fileObj.name)
112 else:
113 logging.getLogger('checkEncoding').debug('checking encoding %s on file object',
114 enc)
115 # try to read the file with the declared encoding
116 fileObj.seek(0)
117 codecs.getreader(enc)(fileObj).read()
118
119
120 ## Make a zip file out of a directory containing python modules
121 def zipdir(directory, no_pyc = False):
122 log = logging.getLogger("zipdir")
123 if not os.path.isdir(directory):
124 raise OSError(20, "Not a directory", directory)
125 msg = "Zipping directory '%s'"
126 if no_pyc:
127 msg += " (without pre-compilation)"
128 log.info(msg, directory)
129 filename = os.path.realpath(directory + ".zip")
130
131 # Open the file in read an update mode
132 if os.path.exists(filename):
133 zipFile = open(filename, "r+b")
134 else:
135 # If the file does not exist, we need to create it.
136 # "append mode" ensures that, in case of two processes trying to
137 # create the file, they do not truncate each other file
138 zipFile = open(filename, "ab")
139
140 locker.lock(zipFile)
141 try:
142 if zipfile.is_zipfile(filename):
143 infolist = zipfile.ZipFile(filename).infolist()
144 else:
145 infolist = []
146 (added, modified, untouched, removed) = _zipChanges(directory, infolist)
147 if added or modified or removed:
148 tempBuf = StringIO()
149 z = zipfile.PyZipFile(tempBuf, "w", zipfile.ZIP_DEFLATED)
150 for f in added + modified + untouched:
151 src = os.path.join(directory, f)
152 checkEncoding(open(src, 'rb'))
153 if no_pyc:
154 log.debug("adding '%s'", f)
155 z.write(src, f)
156 else:
157 # Remove the .pyc file to always force a re-compilation
158 if os.path.exists(src + 'c'):
159 log.debug("removing old .pyc for '%s'", f)
160 os.remove(src + 'c')
161 log.debug("adding '%s'", f)
162 z.writepy(src, os.path.dirname(f))
163 z.close()
164 zipFile.seek(0)
165 zipFile.write(tempBuf.getvalue())
166 zipFile.truncate()
167 log.info("File '%s' closed", filename)
168 else:
169 log.info("Nothing to do on '%s'", filename)
170 except UnicodeDecodeError, x:
171 log.error("Wrong encoding in file '%s':", src)
172 log.error(" %s", x)
173 log.error("Probably you forgot the line '# -*- coding: utf-8 -*-'")
174 sys.exit(1)
175 finally:
176 locker.unlock(zipFile)
177 zipFile.close()
178
179 ## Main function of the script.
180 # Parse arguments and call zipdir() for each directory passed as argument
181 def main(argv = None):
182 from optparse import OptionParser
183 parser = OptionParser(usage = "%prog [options] directory1 [directory2 ...]")
184 parser.add_option("--no-pyc", action = "store_true",
185 help = "copy the .py files without pre-compiling them")
186 parser.add_option("--quiet", action = "store_true",
187 help = "do not print info messages")
188 parser.add_option("--debug", action = "store_true",
189 help = "print debug messages (has priority over --quiet)")
190
191 if argv is None:
192 argv = sys.argv
193 opts, args = parser.parse_args(argv[1:])
194
195 if not args:
196 parser.error("Specify at least one directory to zip")
197
198 # Initialize the logging module
199 level = logging.INFO
200 if opts.quiet:
201 level = logging.WARNING
202 if opts.debug:
203 level = logging.DEBUG
204 logging.basicConfig(level = level)
205
206 if "GAUDI_BUILD_LOCK" in os.environ:
207 _scopedLock = locker.LockFile(os.environ["GAUDI_BUILD_LOCK"], temporary = True)
208 # zip all the directories passed as arguments
209 for d in args:
210 zipdir(d, opts.no_pyc)
211
212 if __name__ == '__main__':
213 main()