crowset.py (8911B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\script\sys\crowset.py 2 from __future__ import print_function 3 import weakref 4 import blue 5 from carbon.common.lib import const 6 ROWSETVERSION = 3 7 8 class CRowset(list): 9 __guid__ = 'dbutil.CRowset' 10 __passbyvalue__ = 1 11 12 def __init__(self, header, rows): 13 list.__init__(self, rows) 14 self.header = header 15 16 def __getslice__(self, i, j): 17 return CRowset(self.header, list.__getslice__(self, i, j)) 18 19 def Copy(self): 20 return CRowset(self.header, self) 21 22 def GetColumnNames(self): 23 return self.header.Keys() 24 25 @property 26 def columns(self): 27 return self.header.Keys() 28 29 def InsertNew(self, listOfValues): 30 self.append(blue.DBRow(self.header, listOfValues)) 31 32 def Sort(self, columnName, caseInsensitive = False, reverse = False): 33 ix = self.header.Keys().index(columnName) 34 if caseInsensitive: 35 Key = lambda a: a[ix].upper() 36 else: 37 Key = lambda a: a[ix] 38 self.sort(key=Key, reverse=reverse) 39 40 def Index(self, columnName): 41 ir = CIndexedRowset(self.header, columnName) 42 ir.Build(self) 43 return ir 44 45 def Filter(self, columnName, indexName = None, allowDuplicateCompoundKeys = False, giveMeSets = False): 46 fr = CFilterRowset(self.header, columnName, indexName, allowDuplicateCompoundKeys, giveMeSets) 47 fr.Build(self) 48 return fr 49 50 def Rebuild(self, rowset): 51 del self[0:len(self)] 52 for row in rowset: 53 self.append(row) 54 55 def Unpack(self, colsNotToUnpack = []): 56 l = [] 57 for row in self: 58 entry = {} 59 for col in self.header.Keys(): 60 if col not in colsNotToUnpack: 61 entry[col] = row[self.header.Index(col)] 62 63 l.append(entry) 64 65 return l 66 67 68 class CIndexedRowset(dict): 69 __guid__ = 'dbutil.CIndexedRowset' 70 71 def __init__(self, header, columnName): 72 self.header = header 73 self.columnName = columnName 74 75 @property 76 def columns(self): 77 return self.header.Keys() 78 79 def Build(self, rowset): 80 if '.' in self.columnName: 81 keys = self.columnName.split('.') 82 c = 0 83 for row in rowset: 84 combinedKey = [] 85 for key in keys: 86 combinedKey.append(row[key]) 87 88 self[tuple(combinedKey)] = row 89 c += 1 90 if c == 25000: 91 c = 0 92 blue.pyos.BeNice() 93 94 else: 95 blue.pyos.XUtil_Index(rowset, self.columnName, self) 96 97 def Rebuild(self, rowset): 98 self.clear() 99 self.Build(rowset) 100 101 102 class CFilterRowset(dict): 103 __guid__ = 'dbutil.CFilterRowset' 104 105 def __init__(self, header, columnName, indexName = None, allowDuplicateCompoundKeys = False, giveMeSets = False): 106 self.header = header 107 self.columnName = columnName 108 self.indexName = indexName 109 self.allowDuplicateCompoundKeys = allowDuplicateCompoundKeys 110 self.giveMeSets = giveMeSets 111 112 @property 113 def columns(self): 114 return self.header.Keys() 115 116 def Build(self, rowset): 117 c = 0 118 keyIdx = self.header.Keys().index(self.columnName) 119 if self.indexName is None: 120 for row in rowset: 121 key = row[keyIdx] 122 if key in self: 123 if self.giveMeSets: 124 self[key].add(row) 125 else: 126 self[key].append(row) 127 elif self.giveMeSets: 128 self[key] = set([row]) 129 else: 130 self[key] = CRowset(row.__header__, [row]) 131 c += 1 132 if c == 10000: 133 c = 0 134 blue.pyos.BeNice() 135 136 else: 137 key2Idx = self.header.Keys().index(self.indexName) 138 for row in rowset: 139 key = row[keyIdx] 140 key2 = row[key2Idx] 141 if key not in self: 142 self[key] = {} 143 if self.allowDuplicateCompoundKeys: 144 if key2 not in self[key]: 145 if self.giveMeSets: 146 self[key][key2] = set() 147 else: 148 self[key][key2] = CRowset(row.__header__, []) 149 if self.giveMeSets: 150 self[key][key2].add(row) 151 else: 152 self[key][key2].append(row) 153 else: 154 self[key][key2] = row 155 c += 1 156 if c == 10000: 157 c = 0 158 blue.pyos.BeNice() 159 160 def Rebuild(self, rowset): 161 self.clear() 162 self.Build(rowset) 163 164 165 def SaveRowset(rs, filename, segmented): 166 print('>>about to SaveRowset', filename, segmented) 167 if isinstance(rs, CRowset): 168 if segmented: 169 raise RuntimeError('SaveRowset: No support for segmented CRowset') 170 elif isinstance(rs, CIndexedRowset): 171 cdata = True 172 elif isinstance(rs, CFilterRowset): 173 cdata = False 174 else: 175 raise RuntimeError("SaveRowset: No support for '%s'" % type(rs)) 176 dataFile = blue.ResFile() 177 if not segmented: 178 dataFile.Create(filename + '.cr') 179 dataFile.Write(blue.marshal.Save((ROWSETVERSION, rs))) 180 else: 181 dataFile.Create(filename + '.data.cr') 182 offsets = {} 183 for k, v in rs.iteritems(): 184 pos = dataFile.pos 185 if cdata: 186 blob = blue.marshal.Save(v.__cdata__) 187 else: 188 blob = blue.marshal.Save(v) 189 dataFile.Write(blob) 190 offsets[k] = (pos, len(blob)) 191 192 indexData = (ROWSETVERSION, 193 rs.header, 194 rs.columnName, 195 offsets) 196 indexFile = blue.ResFile() 197 indexFile.Create(filename + '.index.cr') 198 indexFile.Write(blue.marshal.Save(indexData)) 199 200 201 def LoadRowset(filename): 202 f = blue.ResFile() 203 f.OpenAlways(filename + '.rs.data.cr') 204 version, rs = blue.marshal.Load(f.Read()) 205 if version != ROWSETVERSION: 206 msg = 'DiskData: File %s is version %s, but I want %s' % (filename, version, ROWSETVERSION) 207 raise RuntimeError(msg) 208 return rs 209 210 211 class CRowsetDiskBase(object): 212 __guid__ = 'dbutil.CRowsetDiskBase' 213 __diskRowsets__ = [] 214 215 def __init__(self, filename): 216 if isinstance(self, CIndexedRowset): 217 parent = CIndexedRowset 218 self.cdata = True 219 elif isinstance(self, CFilterRowset): 220 parent = CFilterRowset 221 self.cdata = False 222 else: 223 raise RuntimeError("CRowsetDiskBase: No support for '%s'" % type(self)) 224 f = blue.ResFile() 225 f.OpenAlways(filename + '.index.cr') 226 indexData = blue.marshal.Load(f.Read()) 227 if indexData[0] != ROWSETVERSION: 228 msg = 'DiskData: File %s is version %s, but I want %s' % (filename, indexData.version, ROWSETVERSION) 229 raise RuntimeError(msg) 230 parent.__init__(self, indexData[1], indexData[2]) 231 self.offsets = indexData[3] 232 self.dataFile = blue.ResFile() 233 self.dataFile.OpenAlways(filename + '.data.cr') 234 self.maxUsage = 1 235 self.minAge = 60 236 self.loaded = {} 237 self.__diskRowsets__ = weakref(self) 238 239 def __getitem__(self, key): 240 if key in self.loaded: 241 self.loaded[key][1] = blue.os.GetWallclockTime() 242 self.loaded[key][2] += 1 243 return dict.__getitem__(self, key) 244 pos, size = self.offsets[key] 245 self.dataFile.Seek(pos) 246 blob = self.dataFile.Read(size) 247 data = blue.marshal.Load(blob) 248 if self.cdata: 249 row = blue.DBRow(self.header) 250 row.__cdata__ = data 251 else: 252 row = data 253 self[key] = row 254 self.loaded[key] = [blue.os.GetWallclockTime(), blue.os.GetWallclockTime(), 1] 255 return row 256 257 def __contains__(self, key): 258 return key in self.offsets 259 260 def GetKeys(self): 261 return self.offsets.keys() 262 263 def Flush(self): 264 expire = blue.os.GetWallclockTime() - self.minAge * const.SEC 265 deleted = 0 266 for k, v in self.loaded.items(): 267 if v[2] <= self.maxUsage and v[1] < expire: 268 del self[k] 269 del self.loaded[k] 270 deleted += 1 271 272 return deleted 273 274 275 class CIndexedRowsetDisk(CRowsetDiskBase, CIndexedRowset): 276 __guid__ = 'dbutil.CIndexedRowsetDisk' 277 278 def __init__(self, filename): 279 CRowsetDiskBase.__init__(self, filename) 280 281 282 class CFilterRowsetDisk(CRowsetDiskBase, CFilterRowset): 283 __guid__ = 'dbutil.CFilterRowsetDisk' 284 285 def __init__(self, filename): 286 CRowsetDiskBase.__init__(self, filename)