| The Gaudi Framework v28r2p1 (f1a77ff4)

- Main Page
- Related Pages
- Modules
- Namespaces
- Classes
- Files

- File List
- File Members

ZipPythonDir.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2
3 ## file ZipPythonDir.py
4 # Script to generate a zip file that can replace a directory in the python path.
5
6 import os
7 import sys
8 import zipfile
9 import logging
10 import stat
11 import time
12 import re
13 import codecs
14 from StringIO import StringIO
15
16 ## Class for generic exception coming from the zipdir() function
17 class ZipdirError(RuntimeError):
18 pass
19
20 ## Collect the changes to be applied to the zip file.
21 #
22 # @param directory: directory to be packed in the zip file
23 # @param infolist: list of ZipInfo objects already contained in the zip archive
24 #
25 # @return: tuple of (added, modified, untouched, removed) entries in the directory with respect to the zip file
26 #
27 def _zipChanges(directory, infolist):
28 # gets the dates of the files in the zip archive
29 infos = {}
30 for i in infolist:
31 fn = i.filename
32 if fn.endswith(".pyc"):
33 fn = fn[:-1]
34 infos[fn] = i.date_time
35
36 # gets the changes
37 added = []
38 modified = []
39 untouched = []
40 removed = []
41 all_files = set()
42
43 log = logging.getLogger("zipdir")
44 dirlen = len(directory) + 1
45 for root, dirs, files in os.walk(directory):
46 if "lib-dynload" in dirs:
47 # exclude the directory containing binary modules
48 dirs.remove("lib-dynload")
49 arcdir = root[dirlen:]
50 for f in files:
51 ext = os.path.splitext(f)[1]
52 if ext == ".py": # extensions that can enter the zip file
53 filename = os.path.join(arcdir, f)
54 all_files.add(filename)
55 if filename not in infos:
56 action = "A"
57 added.append(filename)
58 else:
59 filetime = time.localtime(os.stat(os.path.join(directory,filename))[stat.ST_MTIME])[:6]
60 if filetime > infos[filename]:
61 action = "M"
62 modified.append(filename)
63 else:
64 action = "U"
65 untouched.append(filename)
66 if action in ['U']:
67 log.debug(" %s -> %s", action, filename)
68 else:
69 log.info(" %s -> %s", action, filename)
70 # cases that can be ignored
71 elif (ext not in [".pyc", ".pyo", ".stamp", ".cmtref", ".confdb"]
72 and not f.startswith('.__afs')):
73 raise ZipdirError("Cannot add '%s' to the zip file, only '.py' are allowed." % os.path.join(arcdir, f))
74 # check for removed files
75 for filename in infos:
76 if filename not in all_files:
77 removed.append(filename)
78 log.info(" %s -> %s", "R", filename)
79 return (added, modified, untouched, removed)
80
81 def checkEncoding(fileObj):
82 '''
83 Check that a file honors the declared encoding (default ASCII for Python 2
84 and UTF-8 for Python 3).
85
86 Raises a UnicodeDecodeError in case of decoding problems and LookupError if
87 the specified codec does not exists.
88
89 See http://www.python.org/dev/peps/pep-0263/
90 '''
91 from itertools import islice
92
93 # default encoding
94 if sys.version_info[0] <= 2:
95 enc = 'ascii'
96 else:
97 enc = 'utf-8'
98
99 # find the encoding of the file, if specified (in the first two lines)
100 enc_exp = re.compile(r"coding[:=]\s*([-\w.]+)")
101 for l in islice(fileObj, 2):
102 m = enc_exp.search(l)
103 if m:
104 enc = m.group(1)
105 break
106
107 if hasattr(fileObj, 'name'):
108 logging.getLogger('checkEncoding').debug('checking encoding %s on %s',
109 enc, fileObj.name)
110 else:
111 logging.getLogger('checkEncoding').debug('checking encoding %s on file object',
112 enc)
113 # try to read the file with the declared encoding
114 fileObj.seek(0)
115 codecs.getreader(enc)(fileObj).read()
116
117
118 ## Make a zip file out of a directory containing python modules
119 def zipdir(directory, no_pyc = False):
120 filename = os.path.realpath(directory + ".zip")
121 log = logging.getLogger("zipdir")
122 if not os.path.isdir(directory):
123 log.warning('directory %s missing, creating empty .zip file', directory)
124 open(filename, "ab").close()
125 return
126 msg = "Zipping directory '%s'"
127 if no_pyc:
128 msg += " (without pre-compilation)"
129 log.info(msg, directory)
130
131 # Open the file in read an update mode
132 if os.path.exists(filename):
133 zipFile = open(filename, "r+b")
134 else:
135 # If the file does not exist, we need to create it.
136 # "append mode" ensures that, in case of two processes trying to
137 # create the file, they do not truncate each other file
138 zipFile = open(filename, "ab")
139
140 try:
141 if zipfile.is_zipfile(filename):
142 infolist = zipfile.ZipFile(filename).infolist()
143 else:
144 infolist = []
145 (added, modified, untouched, removed) = _zipChanges(directory, infolist)
146 if added or modified or removed:
147 tempBuf = StringIO()
148 z = zipfile.PyZipFile(tempBuf, "w", zipfile.ZIP_DEFLATED)
149 for f in added + modified + untouched:
150 src = os.path.join(directory, f)
151 checkEncoding(open(src, 'rb'))
152 if no_pyc:
153 log.debug("adding '%s'", f)
154 z.write(src, f)
155 else:
156 # Remove the .pyc file to always force a re-compilation
157 if os.path.exists(src + 'c'):
158 log.debug("removing old .pyc for '%s'", f)
159 os.remove(src + 'c')
160 log.debug("adding '%s'", f)
161 z.writepy(src, os.path.dirname(f))
162 z.close()
163 zipFile.seek(0)
164 zipFile.write(tempBuf.getvalue())
165 zipFile.truncate()
166 log.info("File '%s' closed", filename)
167 else:
168 log.info("Nothing to do on '%s'", filename)
169 except UnicodeDecodeError, x:
170 log.error("Wrong encoding in file '%s':", src)
171 log.error(" %s", x)
172 log.error("Probably you forgot the line '# -*- coding: utf-8 -*-'")
173 sys.exit(1)
174 finally:
175 zipFile.close()
176
177 ## Main function of the script.
178 # Parse arguments and call zipdir() for each directory passed as argument
179 def main(argv = None):
180 from optparse import OptionParser
181 parser = OptionParser(usage = "%prog [options] directory1 [directory2 ...]")
182 parser.add_option("--no-pyc", action = "store_true",
183 help = "copy the .py files without pre-compiling them")
184 parser.add_option("--quiet", action = "store_true",
185 help = "do not print info messages")
186 parser.add_option("--debug", action = "store_true",
187 help = "print debug messages (has priority over --quiet)")
188
189 if argv is None:
190 argv = sys.argv
191 opts, args = parser.parse_args(argv[1:])
192
193 if not args:
194 parser.error("Specify at least one directory to zip")
195
196 # Initialize the logging module
197 level = logging.INFO
198 if opts.quiet:
199 level = logging.WARNING
200 if opts.debug:
201 level = logging.DEBUG
202 logging.basicConfig(level=level)
203
204 # zip all the directories passed as arguments
205 for d in args:
206 zipdir(d, opts.no_pyc)
207
208 if __name__ == '__main__':
209 main()
ZipPythonDir.checkEncoding
def checkEncoding(fileObj)
Definition: ZipPythonDir.py:81
hivetimeline.read
def read(f, regex='.*', skipevents=0)
Definition: hivetimeline.py:19
ZipPythonDir.ZipdirError
Class for generic exception coming from the zipdir() function.
Definition: ZipPythonDir.py:17
ZipPythonDir.main
def main(argv=None)
Main function of the script.
Definition: ZipPythonDir.py:179
ZipPythonDir._zipChanges
def _zipChanges(directory, infolist)
Collect the changes to be applied to the zip file.
Definition: ZipPythonDir.py:27
ZipPythonDir.zipdir
def zipdir(directory, no_pyc=False)
Make a zip file out of a directory containing python modules.
Definition: ZipPythonDir.py:119