1#!/usr/bin/env python3 2""" 3# OpenWrt download directory cleanup utility. 4# Delete all but the very last version of the program tarballs. 5# 6# Copyright (C) 2010-2015 Michael Buesch <m@bues.ch> 7# Copyright (C) 2013-2015 OpenWrt.org 8""" 9 10from __future__ import print_function 11 12import sys 13import os 14import re 15import getopt 16 17# Commandline options 18opt_dryrun = False 19 20 21def parseVer_1234(match, filepath): 22 progname = match.group(1) 23 progversion = (int(match.group(2)) << 64) |\ 24 (int(match.group(3)) << 48) |\ 25 (int(match.group(4)) << 32) |\ 26 (int(match.group(5)) << 16) 27 return (progname, progversion) 28 29def parseVer_123(match, filepath): 30 progname = match.group(1) 31 try: 32 patchlevel = match.group(5) 33 except IndexError as e: 34 patchlevel = None 35 if patchlevel: 36 patchlevel = ord(patchlevel[0]) 37 else: 38 patchlevel = 0 39 progversion = (int(match.group(2)) << 64) |\ 40 (int(match.group(3)) << 48) |\ 41 (int(match.group(4)) << 32) |\ 42 patchlevel 43 return (progname, progversion) 44 45def parseVer_12(match, filepath): 46 progname = match.group(1) 47 try: 48 patchlevel = match.group(4) 49 except IndexError as e: 50 patchlevel = None 51 if patchlevel: 52 patchlevel = ord(patchlevel[0]) 53 else: 54 patchlevel = 0 55 progversion = (int(match.group(2)) << 64) |\ 56 (int(match.group(3)) << 48) |\ 57 patchlevel 58 return (progname, progversion) 59 60def parseVer_r(match, filepath): 61 progname = match.group(1) 62 progversion = (int(match.group(2)) << 64) 63 return (progname, progversion) 64 65def parseVer_ymd(match, filepath): 66 progname = match.group(1) 67 progversion = (int(match.group(2)) << 64) |\ 68 (int(match.group(3)) << 48) |\ 69 (int(match.group(4)) << 32) 70 return (progname, progversion) 71 72def parseVer_GIT(match, filepath): 73 progname = match.group(1) 74 st = os.stat(filepath) 75 progversion = int(st.st_mtime) << 64 76 return (progname, progversion) 77 78extensions = ( 79 ".tar.gz", 80 ".tar.bz2", 81 ".tar.xz", 82 ".orig.tar.gz", 83 ".orig.tar.bz2", 84 ".orig.tar.xz", 85 ".zip", 86 ".tgz", 87 ".tbz", 88 ".txz", 89) 90 91versionRegex = ( 92 (re.compile(r"(.+)[-_](\d+)\.(\d+)\.(\d+)\.(\d+)"), parseVer_1234), # xxx-1.2.3.4 93 (re.compile(r"(.+)[-_](\d\d\d\d)-?(\d\d)-?(\d\d)"), parseVer_ymd), # xxx-YYYY-MM-DD 94 (re.compile(r"(.+)[-_]([0-9a-fA-F]{40,40})"), parseVer_GIT), # xxx-GIT_SHASUM 95 (re.compile(r"(.+)[-_](\d+)\.(\d+)\.(\d+)(\w?)"), parseVer_123), # xxx-1.2.3a 96 (re.compile(r"(.+)[-_](\d+)_(\d+)_(\d+)"), parseVer_123), # xxx-1_2_3 97 (re.compile(r"(.+)[-_](\d+)\.(\d+)(\w?)"), parseVer_12), # xxx-1.2a 98 (re.compile(r"(.+)[-_]r?(\d+)"), parseVer_r), # xxx-r1111 99) 100 101blacklist = [ 102 ("linux", re.compile(r"linux-\d.*")), 103 ("gcc", re.compile(r"gcc-.*")), 104 ("wl_apsta", re.compile(r"wl_apsta.*")), 105 (".fw", re.compile(r".*\.fw")), 106 (".arm", re.compile(r".*\.arm")), 107 (".bin", re.compile(r".*\.bin")), 108 ("rt-firmware", re.compile(r"RT[\d\w]+_Firmware.*")), 109] 110 111class EntryParseError(Exception): pass 112 113class Entry: 114 def __init__(self, directory, filename): 115 self.directory = directory 116 self.filename = filename 117 self.progname = "" 118 self.fileext = "" 119 120 for ext in extensions: 121 if filename.endswith(ext): 122 filename = filename[0:0-len(ext)] 123 self.fileext = ext 124 break 125 else: 126 print(self.filename, "has an unknown file-extension") 127 raise EntryParseError("ext") 128 for (regex, parseVersion) in versionRegex: 129 match = regex.match(filename) 130 if match: 131 (self.progname, self.version) = parseVersion( 132 match, directory + "/" + filename + self.fileext) 133 break 134 else: 135 print(self.filename, "has an unknown version pattern") 136 raise EntryParseError("ver") 137 138 def getPath(self): 139 return (self.directory + "/" + self.filename).replace("//", "/") 140 141 def deleteFile(self): 142 path = self.getPath() 143 print("Deleting", path) 144 if not opt_dryrun: 145 os.unlink(path) 146 147 def __ge__(self, y): 148 return self.version >= y.version 149 150def usage(): 151 print("OpenWrt download directory cleanup utility") 152 print("Usage: " + sys.argv[0] + " [OPTIONS] <path/to/dl>") 153 print("") 154 print(" -d|--dry-run Do a dry-run. Don't delete any files") 155 print(" -B|--show-blacklist Show the blacklist and exit") 156 print(" -w|--whitelist ITEM Remove ITEM from blacklist") 157 158def main(argv): 159 global opt_dryrun 160 161 try: 162 (opts, args) = getopt.getopt(argv[1:], 163 "hdBw:", 164 [ "help", "dry-run", "show-blacklist", "whitelist=", ]) 165 if len(args) != 1: 166 usage() 167 return 1 168 except getopt.GetoptError as e: 169 usage() 170 return 1 171 directory = args[0] 172 for (o, v) in opts: 173 if o in ("-h", "--help"): 174 usage() 175 return 0 176 if o in ("-d", "--dry-run"): 177 opt_dryrun = True 178 if o in ("-w", "--whitelist"): 179 for i in range(0, len(blacklist)): 180 (name, regex) = blacklist[i] 181 if name == v: 182 del blacklist[i] 183 break 184 else: 185 print("Whitelist error: Item", v,\ 186 "is not in blacklist") 187 return 1 188 if o in ("-B", "--show-blacklist"): 189 for (name, regex) in blacklist: 190 sep = "\t\t" 191 if len(name) >= 8: 192 sep = "\t" 193 print("%s%s(%s)" % (name, sep, regex.pattern)) 194 return 0 195 196 # Create a directory listing and parse the file names. 197 entries = [] 198 for filename in os.listdir(directory): 199 if filename == "." or filename == "..": 200 continue 201 for (name, regex) in blacklist: 202 if regex.match(filename): 203 if opt_dryrun: 204 print(filename, "is blacklisted") 205 break 206 else: 207 try: 208 entries.append(Entry(directory, filename)) 209 except EntryParseError as e: 210 pass 211 212 # Create a map of programs 213 progmap = {} 214 for entry in entries: 215 if entry.progname in progmap.keys(): 216 progmap[entry.progname].append(entry) 217 else: 218 progmap[entry.progname] = [entry,] 219 220 # Traverse the program map and delete everything but the last version 221 for prog in progmap: 222 lastVersion = None 223 versions = progmap[prog] 224 for version in versions: 225 if lastVersion is None or version >= lastVersion: 226 lastVersion = version 227 if lastVersion: 228 for version in versions: 229 if version is not lastVersion: 230 version.deleteFile() 231 if opt_dryrun: 232 print("Keeping", lastVersion.getPath()) 233 234 return 0 235 236if __name__ == "__main__": 237 sys.exit(main(sys.argv)) 238