Add hash of SDE tool for the bots.
1de4bdf147a8a3b93306ecc65cf1f15a8334f508a29ab0b3ee163443ce764d5e sde-external-7.58.0-2017-01-23-lin.tar.bz2
Also teach extract.py to handle symlinks and tar.bz2 files.
BUG=180
Change-Id: Iddce09169f077c8f1bef9bd0dd0e05605b17769a
Reviewed-on: https://boringssl-review.googlesource.com/16244
Reviewed-by: Adam Langley <agl@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
diff --git a/util/bot/extract.py b/util/bot/extract.py
index e36ce99..4680cfe 100644
--- a/util/bot/extract.py
+++ b/util/bot/extract.py
@@ -37,30 +37,47 @@
return os.path.join(output, path)
+class FileEntry(object):
+ def __init__(self, path, mode, fileobj):
+ self.path = path
+ self.mode = mode
+ self.fileobj = fileobj
+
+
+class SymlinkEntry(object):
+ def __init__(self, path, mode, target):
+ self.path = path
+ self.mode = mode
+ self.target = target
+
+
def IterateZip(path):
"""
- IterateZip opens the zip file at path and returns a generator of
- (filename, mode, fileobj) tuples for each file in it.
+ IterateZip opens the zip file at path and returns a generator of entry objects
+ for each file in it.
"""
with zipfile.ZipFile(path, 'r') as zip_file:
for info in zip_file.infolist():
if info.filename.endswith('/'):
continue
- yield (info.filename, None, zip_file.open(info))
+ yield FileEntry(info.filename, None, zip_file.open(info))
-def IterateTar(path):
+def IterateTar(path, compression):
"""
- IterateTar opens the tar.gz file at path and returns a generator of
- (filename, mode, fileobj) tuples for each file in it.
+ IterateTar opens the tar.gz or tar.bz2 file at path and returns a generator of
+ entry objects for each file in it.
"""
- with tarfile.open(path, 'r:gz') as tar_file:
+ with tarfile.open(path, 'r:' + compression) as tar_file:
for info in tar_file:
if info.isdir():
- continue
- if not info.isfile():
+ pass
+ elif info.issym():
+ yield SymlinkEntry(info.name, None, info.linkname)
+ elif info.isfile():
+ yield FileEntry(info.name, info.mode, tar_file.extractfile(info))
+ else:
raise ValueError('Unknown entry type "%s"' % (info.name, ))
- yield (info.name, info.mode, tar_file.extractfile(info))
def main(args):
@@ -98,7 +115,9 @@
if archive.endswith('.zip'):
entries = IterateZip(archive)
elif archive.endswith('.tar.gz'):
- entries = IterateTar(archive)
+ entries = IterateTar(archive, 'gz')
+ elif archive.endswith('.tar.bz2'):
+ entries = IterateTar(archive, 'bz2')
else:
raise ValueError(archive)
@@ -110,13 +129,13 @@
print "Extracting %s to %s" % (archive, output)
prefix = None
num_extracted = 0
- for path, mode, inp in entries:
+ for entry in entries:
# Even on Windows, zip files must always use forward slashes.
- if '\\' in path or path.startswith('/'):
- raise ValueError(path)
+ if '\\' in entry.path or entry.path.startswith('/'):
+ raise ValueError(entry.path)
if not options.no_prefix:
- new_prefix, rest = path.split('/', 1)
+ new_prefix, rest = entry.path.split('/', 1)
# Ensure the archive is consistent.
if prefix is None:
@@ -124,20 +143,25 @@
if prefix != new_prefix:
raise ValueError((prefix, new_prefix))
else:
- rest = path
+ rest = entry.path
# Extract the file into the output directory.
fixed_path = CheckedJoin(output, rest)
if not os.path.isdir(os.path.dirname(fixed_path)):
os.makedirs(os.path.dirname(fixed_path))
- with open(fixed_path, 'wb') as out:
- shutil.copyfileobj(inp, out)
+ if isinstance(entry, FileEntry):
+ with open(fixed_path, 'wb') as out:
+ shutil.copyfileobj(entry.fileobj, out)
+ elif isinstance(entry, SymlinkEntry):
+ os.symlink(entry.target, fixed_path)
+ else:
+ raise TypeError('unknown entry type')
# Fix up permissions if needbe.
# TODO(davidben): To be extra tidy, this should only track the execute bit
# as in git.
- if mode is not None:
- os.chmod(fixed_path, mode)
+ if entry.mode is not None:
+ os.chmod(fixed_path, entry.mode)
# Print every 100 files, so bots do not time out on large archives.
num_extracted += 1
diff --git a/util/bot/sde-linux64.tar.bz2.sha1 b/util/bot/sde-linux64.tar.bz2.sha1
new file mode 100644
index 0000000..afa9875
--- /dev/null
+++ b/util/bot/sde-linux64.tar.bz2.sha1
@@ -0,0 +1 @@
+3d612f0095a0830656614e5b442f70289e266088
\ No newline at end of file
