sphinxapi.py (29221B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\carbon\common\lib\sphinxapi.py 2 import socket 3 import re 4 from struct import * 5 SEARCHD_COMMAND_SEARCH = 0 6 SEARCHD_COMMAND_EXCERPT = 1 7 SEARCHD_COMMAND_UPDATE = 2 8 SEARCHD_COMMAND_KEYWORDS = 3 9 SEARCHD_COMMAND_PERSIST = 4 10 SEARCHD_COMMAND_STATUS = 5 11 SEARCHD_COMMAND_FLUSHATTRS = 7 12 VER_COMMAND_SEARCH = 285 13 VER_COMMAND_EXCERPT = 260 14 VER_COMMAND_UPDATE = 259 15 VER_COMMAND_KEYWORDS = 256 16 VER_COMMAND_STATUS = 256 17 VER_COMMAND_FLUSHATTRS = 256 18 SEARCHD_OK = 0 19 SEARCHD_ERROR = 1 20 SEARCHD_RETRY = 2 21 SEARCHD_WARNING = 3 22 SPH_MATCH_ALL = 0 23 SPH_MATCH_ANY = 1 24 SPH_MATCH_PHRASE = 2 25 SPH_MATCH_BOOLEAN = 3 26 SPH_MATCH_EXTENDED = 4 27 SPH_MATCH_FULLSCAN = 5 28 SPH_MATCH_EXTENDED2 = 6 29 SPH_RANK_PROXIMITY_BM25 = 0 30 SPH_RANK_BM25 = 1 31 SPH_RANK_NONE = 2 32 SPH_RANK_WORDCOUNT = 3 33 SPH_RANK_PROXIMITY = 4 34 SPH_RANK_MATCHANY = 5 35 SPH_RANK_FIELDMASK = 6 36 SPH_RANK_SPH04 = 7 37 SPH_RANK_EXPR = 8 38 SPH_RANK_TOTAL = 9 39 SPH_SORT_RELEVANCE = 0 40 SPH_SORT_ATTR_DESC = 1 41 SPH_SORT_ATTR_ASC = 2 42 SPH_SORT_TIME_SEGMENTS = 3 43 SPH_SORT_EXTENDED = 4 44 SPH_SORT_EXPR = 5 45 SPH_FILTER_VALUES = 0 46 SPH_FILTER_RANGE = 1 47 SPH_FILTER_FLOATRANGE = 2 48 SPH_ATTR_NONE = 0 49 SPH_ATTR_INTEGER = 1 50 SPH_ATTR_TIMESTAMP = 2 51 SPH_ATTR_ORDINAL = 3 52 SPH_ATTR_BOOL = 4 53 SPH_ATTR_FLOAT = 5 54 SPH_ATTR_BIGINT = 6 55 SPH_ATTR_STRING = 7 56 SPH_ATTR_FACTORS = 1001 57 SPH_ATTR_MULTI = 1073741825L 58 SPH_ATTR_MULTI64 = 1073741826L 59 SPH_ATTR_TYPES = (SPH_ATTR_NONE, 60 SPH_ATTR_INTEGER, 61 SPH_ATTR_TIMESTAMP, 62 SPH_ATTR_ORDINAL, 63 SPH_ATTR_BOOL, 64 SPH_ATTR_FLOAT, 65 SPH_ATTR_BIGINT, 66 SPH_ATTR_STRING, 67 SPH_ATTR_MULTI, 68 SPH_ATTR_MULTI64) 69 SPH_GROUPBY_DAY = 0 70 SPH_GROUPBY_WEEK = 1 71 SPH_GROUPBY_MONTH = 2 72 SPH_GROUPBY_YEAR = 3 73 SPH_GROUPBY_ATTR = 4 74 SPH_GROUPBY_ATTRPAIR = 5 75 76 class SphinxClient(): 77 78 def __init__(self): 79 self._host = 'localhost' 80 self._port = 9312 81 self._path = None 82 self._socket = None 83 self._offset = 0 84 self._limit = 20 85 self._mode = SPH_MATCH_EXTENDED2 86 self._weights = [] 87 self._sort = SPH_SORT_RELEVANCE 88 self._sortby = '' 89 self._min_id = 0 90 self._max_id = 0 91 self._filters = [] 92 self._groupby = '' 93 self._groupfunc = SPH_GROUPBY_DAY 94 self._groupsort = '@group desc' 95 self._groupdistinct = '' 96 self._maxmatches = 100 97 self._cutoff = 0 98 self._retrycount = 0 99 self._retrydelay = 0 100 self._anchor = {} 101 self._indexweights = {} 102 self._ranker = SPH_RANK_PROXIMITY_BM25 103 self._rankexpr = '' 104 self._maxquerytime = 0 105 self._timeout = 1.0 106 self._fieldweights = {} 107 self._overrides = {} 108 self._select = '*' 109 self._query_flags = 0 110 self._predictedtime = 0 111 self._outerorderby = '' 112 self._outeroffset = 0 113 self._outerlimit = 0 114 self._hasouter = False 115 self._error = '' 116 self._warning = '' 117 self._reqs = [] 118 119 def __del__(self): 120 if self._socket: 121 self._socket.close() 122 123 def GetLastError(self): 124 return self._error 125 126 def GetLastWarning(self): 127 return self._warning 128 129 def SetServer(self, host, port = None): 130 if host.startswith('/'): 131 self._path = host 132 return 133 if host.startswith('unix://'): 134 self._path = host[7:] 135 return 136 self._host = host 137 if isinstance(port, int): 138 self._port = port 139 self._path = None 140 141 def SetConnectTimeout(self, timeout): 142 self._timeout = max(0.001, timeout) 143 144 def _Connect(self): 145 if self._socket: 146 self._socket.close() 147 self._socket = None 148 try: 149 if self._path: 150 af = socket.AF_UNIX 151 addr = self._path 152 desc = self._path 153 else: 154 af = socket.AF_INET 155 addr = (self._host, self._port) 156 desc = '%s;%s' % addr 157 sock = socket.socket(af, socket.SOCK_STREAM) 158 sock.settimeout(self._timeout) 159 sock.connect(addr) 160 except socket.error as msg: 161 if sock: 162 sock.close() 163 self._error = 'connection to %s failed (%s)' % (desc, msg) 164 return 165 166 v = unpack('>L', sock.recv(4)) 167 if v < 1: 168 sock.close() 169 self._error = 'expected searchd protocol version, got %s' % v 170 return 171 sock.send(pack('>L', 1)) 172 return sock 173 174 def _GetResponse(self, sock, client_ver): 175 status, ver, length = unpack('>2HL', sock.recv(8)) 176 response = '' 177 left = length 178 while left > 0: 179 chunk = sock.recv(left) 180 if chunk: 181 response += chunk 182 left -= len(chunk) 183 else: 184 break 185 186 if not self._socket: 187 sock.close() 188 read = len(response) 189 if not response or read != length: 190 if length: 191 self._error = 'failed to read searchd response (status=%s, ver=%s, len=%s, read=%s)' % (status, 192 ver, 193 length, 194 read) 195 else: 196 self._error = 'received zero-sized searchd response' 197 return None 198 if status == SEARCHD_WARNING: 199 wend = 4 + unpack('>L', response[0:4])[0] 200 self._warning = response[4:wend] 201 return response[wend:] 202 if status == SEARCHD_ERROR: 203 self._error = 'searchd error: ' + response[4:] 204 return None 205 if status == SEARCHD_RETRY: 206 self._error = 'temporary searchd error: ' + response[4:] 207 return None 208 if status != SEARCHD_OK: 209 self._error = 'unknown status code %d' % status 210 return None 211 if ver < client_ver: 212 self._warning = "searchd command v.%d.%d older than client's v.%d.%d, some options might not work" % (ver >> 8, 213 ver & 255, 214 client_ver >> 8, 215 client_ver & 255) 216 return response 217 218 def _Send(self, sock, req): 219 total = 0 220 while True: 221 sent = sock.send(req[total:]) 222 if sent <= 0: 223 break 224 total = total + sent 225 226 return total 227 228 def SetLimits(self, offset, limit, maxmatches = 0, cutoff = 0): 229 self._offset = offset 230 self._limit = limit 231 if maxmatches > 0: 232 self._maxmatches = maxmatches 233 if cutoff >= 0: 234 self._cutoff = cutoff 235 236 def SetMaxQueryTime(self, maxquerytime): 237 self._maxquerytime = maxquerytime 238 239 def SetMatchMode(self, mode): 240 self._mode = mode 241 242 def SetRankingMode(self, ranker, rankexpr = ''): 243 self._ranker = ranker 244 self._rankexpr = rankexpr 245 246 def SetSortMode(self, mode, clause = ''): 247 self._sort = mode 248 self._sortby = clause 249 250 def SetWeights(self, weights): 251 for w in weights: 252 AssertUInt32(w) 253 254 self._weights = weights 255 256 def SetFieldWeights(self, weights): 257 for key, val in weights.items(): 258 AssertUInt32(val) 259 260 self._fieldweights = weights 261 262 def SetIndexWeights(self, weights): 263 for key, val in weights.items(): 264 AssertUInt32(val) 265 266 self._indexweights = weights 267 268 def SetIDRange(self, minid, maxid): 269 self._min_id = minid 270 self._max_id = maxid 271 272 def SetFilter(self, attribute, values, exclude = 0): 273 for value in values: 274 AssertInt32(value) 275 276 self._filters.append({'type': SPH_FILTER_VALUES, 277 'attr': attribute, 278 'exclude': exclude, 279 'values': values}) 280 281 def SetFilterRange(self, attribute, min_, max_, exclude = 0): 282 AssertInt32(min_) 283 AssertInt32(max_) 284 self._filters.append({'type': SPH_FILTER_RANGE, 285 'attr': attribute, 286 'exclude': exclude, 287 'min': min_, 288 'max': max_}) 289 290 def SetFilterFloatRange(self, attribute, min_, max_, exclude = 0): 291 self._filters.append({'type': SPH_FILTER_FLOATRANGE, 292 'attr': attribute, 293 'exclude': exclude, 294 'min': min_, 295 'max': max_}) 296 297 def SetGeoAnchor(self, attrlat, attrlong, latitude, longitude): 298 self._anchor['attrlat'] = attrlat 299 self._anchor['attrlong'] = attrlong 300 self._anchor['lat'] = latitude 301 self._anchor['long'] = longitude 302 303 def SetGroupBy(self, attribute, func, groupsort = '@group desc'): 304 self._groupby = attribute 305 self._groupfunc = func 306 self._groupsort = groupsort 307 308 def SetGroupDistinct(self, attribute): 309 self._groupdistinct = attribute 310 311 def SetRetries(self, count, delay = 0): 312 self._retrycount = count 313 self._retrydelay = delay 314 315 def SetOverride(self, name, type, values): 316 self._overrides[name] = {'name': name, 317 'type': type, 318 'values': values} 319 320 def SetSelect(self, select): 321 self._select = select 322 323 def SetQueryFlag(self, name, value): 324 known_names = ['reverse_scan', 325 'sort_method', 326 'max_predicted_time', 327 'boolean_simplify', 328 'idf'] 329 flags = {'reverse_scan': [0, 1], 330 'sort_method': ['pq', 'kbuffer'], 331 'max_predicted_time': [0], 332 'boolean_simplify': [True, False], 333 'idf': ['normalized', 'plain']} 334 if name == 'reverse_scan': 335 self._query_flags = SetBit(self._query_flags, 0, value == 1) 336 if name == 'sort_method': 337 self._query_flags = SetBit(self._query_flags, 1, value == 'kbuffer') 338 if name == 'max_predicted_time': 339 self._query_flags = SetBit(self._query_flags, 2, value > 0) 340 self._predictedtime = int(value) 341 if name == 'boolean_simplify': 342 self._query_flags = SetBit(self._query_flags, 3, value) 343 if name == 'idf': 344 self._query_flags = SetBit(self._query_flags, 4, value == 'plain') 345 346 def SetOuterSelect(self, orderby, offset, limit): 347 self._outerorderby = orderby 348 self._outeroffset = offset 349 self._outerlimit = limit 350 self._hasouter = True 351 352 def ResetOverrides(self): 353 self._overrides = {} 354 355 def ResetFilters(self): 356 self._filters = [] 357 self._anchor = {} 358 359 def ResetGroupBy(self): 360 self._groupby = '' 361 self._groupfunc = SPH_GROUPBY_DAY 362 self._groupsort = '@group desc' 363 self._groupdistinct = '' 364 365 def ResetQueryFlag(self): 366 self._query_flags = 0 367 self._predictedtime = 0 368 369 def ResetOuterSelect(self): 370 self._outerorderby = '' 371 self._outeroffset = 0 372 self._outerlimit = 0 373 self._hasouter = False 374 375 def Query(self, query, index = '*', comment = ''): 376 self.AddQuery(query, index, comment) 377 results = self.RunQueries() 378 self._reqs = [] 379 if not results or len(results) == 0: 380 return None 381 self._error = results[0]['error'] 382 self._warning = results[0]['warning'] 383 if results[0]['status'] == SEARCHD_ERROR: 384 return None 385 return results[0] 386 387 def AddQuery(self, query, index = '*', comment = ''): 388 req = [] 389 req.append(pack('>5L', self._query_flags, self._offset, self._limit, self._mode, self._ranker)) 390 if self._ranker == SPH_RANK_EXPR: 391 req.append(pack('>L', len(self._rankexpr))) 392 req.append(self._rankexpr) 393 req.append(pack('>L', self._sort)) 394 req.append(pack('>L', len(self._sortby))) 395 req.append(self._sortby) 396 if isinstance(query, unicode): 397 query = query.encode('utf-8') 398 req.append(pack('>L', len(query))) 399 req.append(query) 400 req.append(pack('>L', len(self._weights))) 401 for w in self._weights: 402 req.append(pack('>L', w)) 403 404 req.append(pack('>L', len(index))) 405 req.append(index) 406 req.append(pack('>L', 1)) 407 req.append(pack('>Q', self._min_id)) 408 req.append(pack('>Q', self._max_id)) 409 req.append(pack('>L', len(self._filters))) 410 for f in self._filters: 411 req.append(pack('>L', len(f['attr'])) + f['attr']) 412 filtertype = f['type'] 413 req.append(pack('>L', filtertype)) 414 if filtertype == SPH_FILTER_VALUES: 415 req.append(pack('>L', len(f['values']))) 416 for val in f['values']: 417 req.append(pack('>q', val)) 418 419 elif filtertype == SPH_FILTER_RANGE: 420 req.append(pack('>2q', f['min'], f['max'])) 421 elif filtertype == SPH_FILTER_FLOATRANGE: 422 req.append(pack('>2f', f['min'], f['max'])) 423 req.append(pack('>L', f['exclude'])) 424 425 req.append(pack('>2L', self._groupfunc, len(self._groupby))) 426 req.append(self._groupby) 427 req.append(pack('>2L', self._maxmatches, len(self._groupsort))) 428 req.append(self._groupsort) 429 req.append(pack('>LLL', self._cutoff, self._retrycount, self._retrydelay)) 430 req.append(pack('>L', len(self._groupdistinct))) 431 req.append(self._groupdistinct) 432 if len(self._anchor) == 0: 433 req.append(pack('>L', 0)) 434 else: 435 attrlat, attrlong = self._anchor['attrlat'], self._anchor['attrlong'] 436 latitude, longitude = self._anchor['lat'], self._anchor['long'] 437 req.append(pack('>L', 1)) 438 req.append(pack('>L', len(attrlat)) + attrlat) 439 req.append(pack('>L', len(attrlong)) + attrlong) 440 req.append(pack('>f', latitude) + pack('>f', longitude)) 441 req.append(pack('>L', len(self._indexweights))) 442 for indx, weight in self._indexweights.items(): 443 req.append(pack('>L', len(indx)) + indx + pack('>L', weight)) 444 445 req.append(pack('>L', self._maxquerytime)) 446 req.append(pack('>L', len(self._fieldweights))) 447 for field, weight in self._fieldweights.items(): 448 req.append(pack('>L', len(field)) + field + pack('>L', weight)) 449 450 comment = str(comment) 451 req.append(pack('>L', len(comment)) + comment) 452 req.append(pack('>L', len(self._overrides))) 453 for v in self._overrides.values(): 454 req.extend((pack('>L', len(v['name'])), v['name'])) 455 req.append(pack('>LL', v['type'], len(v['values']))) 456 for id, value in v['values'].iteritems(): 457 req.append(pack('>Q', id)) 458 if v['type'] == SPH_ATTR_FLOAT: 459 req.append(pack('>f', value)) 460 elif v['type'] == SPH_ATTR_BIGINT: 461 req.append(pack('>q', value)) 462 else: 463 req.append(pack('>l', value)) 464 465 req.append(pack('>L', len(self._select))) 466 req.append(self._select) 467 if self._predictedtime > 0: 468 req.append(pack('>L', self._predictedtime)) 469 req.append(pack('>L', len(self._outerorderby)) + self._outerorderby) 470 req.append(pack('>2L', self._outeroffset, self._outerlimit)) 471 if self._hasouter: 472 req.append(pack('>L', 1)) 473 else: 474 req.append(pack('>L', 0)) 475 req = ''.join(req) 476 self._reqs.append(req) 477 478 def RunQueries(self): 479 if len(self._reqs) == 0: 480 self._error = 'no queries defined, issue AddQuery() first' 481 return None 482 sock = self._Connect() 483 if not sock: 484 return None 485 req = ''.join(self._reqs) 486 length = len(req) + 8 487 req = pack('>HHLLL', SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, length, 0, len(self._reqs)) + req 488 self._Send(sock, req) 489 response = self._GetResponse(sock, VER_COMMAND_SEARCH) 490 if not response: 491 return None 492 nreqs = len(self._reqs) 493 max_ = len(response) 494 p = 0 495 results = [] 496 for i in range(0, nreqs, 1): 497 result = {} 498 results.append(result) 499 result['error'] = '' 500 result['warning'] = '' 501 status = unpack('>L', response[p:p + 4])[0] 502 p += 4 503 result['status'] = status 504 if status != SEARCHD_OK: 505 length = unpack('>L', response[p:p + 4])[0] 506 p += 4 507 message = response[p:p + length] 508 p += length 509 if status == SEARCHD_WARNING: 510 result['warning'] = message 511 else: 512 result['error'] = message 513 continue 514 fields = [] 515 attrs = [] 516 nfields = unpack('>L', response[p:p + 4])[0] 517 p += 4 518 while nfields > 0 and p < max_: 519 nfields -= 1 520 length = unpack('>L', response[p:p + 4])[0] 521 p += 4 522 fields.append(response[p:p + length]) 523 p += length 524 525 result['fields'] = fields 526 nattrs = unpack('>L', response[p:p + 4])[0] 527 p += 4 528 while nattrs > 0 and p < max_: 529 nattrs -= 1 530 length = unpack('>L', response[p:p + 4])[0] 531 p += 4 532 attr = response[p:p + length] 533 p += length 534 type_ = unpack('>L', response[p:p + 4])[0] 535 p += 4 536 attrs.append([attr, type_]) 537 538 result['attrs'] = attrs 539 count = unpack('>L', response[p:p + 4])[0] 540 p += 4 541 id64 = unpack('>L', response[p:p + 4])[0] 542 p += 4 543 result['matches'] = [] 544 while count > 0 and p < max_: 545 count -= 1 546 if id64: 547 doc, weight = unpack('>QL', response[p:p + 12]) 548 p += 12 549 else: 550 doc, weight = unpack('>2L', response[p:p + 8]) 551 p += 8 552 match = {'id': doc, 553 'weight': weight, 554 'attrs': {}} 555 for i in range(len(attrs)): 556 if attrs[i][1] == SPH_ATTR_FLOAT: 557 match['attrs'][attrs[i][0]] = unpack('>f', response[p:p + 4])[0] 558 elif attrs[i][1] == SPH_ATTR_BIGINT: 559 match['attrs'][attrs[i][0]] = unpack('>q', response[p:p + 8])[0] 560 p += 4 561 elif attrs[i][1] == SPH_ATTR_STRING: 562 slen = unpack('>L', response[p:p + 4])[0] 563 p += 4 564 match['attrs'][attrs[i][0]] = '' 565 if slen > 0: 566 match['attrs'][attrs[i][0]] = response[p:p + slen] 567 p += slen - 4 568 elif attrs[i][1] == SPH_ATTR_FACTORS: 569 slen = unpack('>L', response[p:p + 4])[0] 570 p += 4 571 match['attrs'][attrs[i][0]] = '' 572 if slen > 0: 573 match['attrs'][attrs[i][0]] = response[p:p + slen - 4] 574 p += slen - 4 575 p -= 4 576 elif attrs[i][1] == SPH_ATTR_MULTI: 577 match['attrs'][attrs[i][0]] = [] 578 nvals = unpack('>L', response[p:p + 4])[0] 579 p += 4 580 for n in range(0, nvals, 1): 581 match['attrs'][attrs[i][0]].append(unpack('>L', response[p:p + 4])[0]) 582 p += 4 583 584 p -= 4 585 elif attrs[i][1] == SPH_ATTR_MULTI64: 586 match['attrs'][attrs[i][0]] = [] 587 nvals = unpack('>L', response[p:p + 4])[0] 588 nvals = nvals / 2 589 p += 4 590 for n in range(0, nvals, 1): 591 match['attrs'][attrs[i][0]].append(unpack('>q', response[p:p + 8])[0]) 592 p += 8 593 594 p -= 4 595 else: 596 match['attrs'][attrs[i][0]] = unpack('>L', response[p:p + 4])[0] 597 p += 4 598 599 result['matches'].append(match) 600 601 result['total'], result['total_found'], result['time'], words = unpack('>4L', response[p:p + 16]) 602 result['time'] = '%.3f' % (result['time'] / 1000.0) 603 p += 16 604 result['words'] = [] 605 while words > 0: 606 words -= 1 607 length = unpack('>L', response[p:p + 4])[0] 608 p += 4 609 word = response[p:p + length] 610 p += length 611 docs, hits = unpack('>2L', response[p:p + 8]) 612 p += 8 613 result['words'].append({'word': word, 614 'docs': docs, 615 'hits': hits}) 616 617 self._reqs = [] 618 return results 619 620 def BuildExcerpts(self, docs, index, words, opts = None): 621 if not opts: 622 opts = {} 623 if isinstance(words, unicode): 624 words = words.encode('utf-8') 625 sock = self._Connect() 626 if not sock: 627 return None 628 opts.setdefault('before_match', '<b>') 629 opts.setdefault('after_match', '</b>') 630 opts.setdefault('chunk_separator', ' ... ') 631 opts.setdefault('html_strip_mode', 'index') 632 opts.setdefault('limit', 256) 633 opts.setdefault('limit_passages', 0) 634 opts.setdefault('limit_words', 0) 635 opts.setdefault('around', 5) 636 opts.setdefault('start_passage_id', 1) 637 opts.setdefault('passage_boundary', 'none') 638 flags = 1 639 if opts.get('exact_phrase'): 640 flags |= 2 641 if opts.get('single_passage'): 642 flags |= 4 643 if opts.get('use_boundaries'): 644 flags |= 8 645 if opts.get('weight_order'): 646 flags |= 16 647 if opts.get('query_mode'): 648 flags |= 32 649 if opts.get('force_all_words'): 650 flags |= 64 651 if opts.get('load_files'): 652 flags |= 128 653 if opts.get('allow_empty'): 654 flags |= 256 655 if opts.get('emit_zones'): 656 flags |= 512 657 if opts.get('load_files_scattered'): 658 flags |= 1024 659 req = [pack('>2L', 0, flags)] 660 req.append(pack('>L', len(index))) 661 req.append(index) 662 req.append(pack('>L', len(words))) 663 req.append(words) 664 req.append(pack('>L', len(opts['before_match']))) 665 req.append(opts['before_match']) 666 req.append(pack('>L', len(opts['after_match']))) 667 req.append(opts['after_match']) 668 req.append(pack('>L', len(opts['chunk_separator']))) 669 req.append(opts['chunk_separator']) 670 req.append(pack('>L', int(opts['limit']))) 671 req.append(pack('>L', int(opts['around']))) 672 req.append(pack('>L', int(opts['limit_passages']))) 673 req.append(pack('>L', int(opts['limit_words']))) 674 req.append(pack('>L', int(opts['start_passage_id']))) 675 req.append(pack('>L', len(opts['html_strip_mode']))) 676 req.append(opts['html_strip_mode']) 677 req.append(pack('>L', len(opts['passage_boundary']))) 678 req.append(opts['passage_boundary']) 679 req.append(pack('>L', len(docs))) 680 for doc in docs: 681 if isinstance(doc, unicode): 682 doc = doc.encode('utf-8') 683 req.append(pack('>L', len(doc))) 684 req.append(doc) 685 686 req = ''.join(req) 687 length = len(req) 688 req = pack('>2HL', SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, length) + req 689 self._Send(sock, req) 690 response = self._GetResponse(sock, VER_COMMAND_EXCERPT) 691 if not response: 692 return [] 693 pos = 0 694 res = [] 695 rlen = len(response) 696 for i in range(len(docs)): 697 length = unpack('>L', response[pos:pos + 4])[0] 698 pos += 4 699 if pos + length > rlen: 700 self._error = 'incomplete reply' 701 return [] 702 res.append(response[pos:pos + length]) 703 pos += length 704 705 return res 706 707 def UpdateAttributes(self, index, attrs, values, mva = False, ignorenonexistent = False): 708 for attr in attrs: 709 pass 710 711 for docid, entry in values.items(): 712 AssertUInt32(docid) 713 for val in entry: 714 if mva: 715 for vals in val: 716 AssertInt32(vals) 717 718 else: 719 AssertInt32(val) 720 721 req = [pack('>L', len(index)), index] 722 req.append(pack('>L', len(attrs))) 723 ignore_absent = 0 724 if ignorenonexistent: 725 ignore_absent = 1 726 req.append(pack('>L', ignore_absent)) 727 mva_attr = 0 728 if mva: 729 mva_attr = 1 730 for attr in attrs: 731 req.append(pack('>L', len(attr)) + attr) 732 req.append(pack('>L', mva_attr)) 733 734 req.append(pack('>L', len(values))) 735 for docid, entry in values.items(): 736 req.append(pack('>Q', docid)) 737 for val in entry: 738 val_len = val 739 if mva: 740 val_len = len(val) 741 req.append(pack('>L', val_len)) 742 if mva: 743 for vals in val: 744 req.append(pack('>L', vals)) 745 746 sock = self._Connect() 747 if not sock: 748 return None 749 req = ''.join(req) 750 length = len(req) 751 req = pack('>2HL', SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, length) + req 752 self._Send(sock, req) 753 response = self._GetResponse(sock, VER_COMMAND_UPDATE) 754 if not response: 755 return -1 756 updated = unpack('>L', response[0:4])[0] 757 return updated 758 759 def BuildKeywords(self, query, index, hits): 760 req = [pack('>L', len(query)) + query] 761 req.append(pack('>L', len(index)) + index) 762 req.append(pack('>L', hits)) 763 sock = self._Connect() 764 if not sock: 765 return None 766 req = ''.join(req) 767 length = len(req) 768 req = pack('>2HL', SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, length) + req 769 self._Send(sock, req) 770 response = self._GetResponse(sock, VER_COMMAND_KEYWORDS) 771 if not response: 772 return None 773 res = [] 774 nwords = unpack('>L', response[0:4])[0] 775 p = 4 776 max_ = len(response) 777 while nwords > 0 and p < max_: 778 nwords -= 1 779 length = unpack('>L', response[p:p + 4])[0] 780 p += 4 781 tokenized = response[p:p + length] 782 p += length 783 length = unpack('>L', response[p:p + 4])[0] 784 p += 4 785 normalized = response[p:p + length] 786 p += length 787 entry = {'tokenized': tokenized, 788 'normalized': normalized} 789 if hits: 790 entry['docs'], entry['hits'] = unpack('>2L', response[p:p + 8]) 791 p += 8 792 res.append(entry) 793 794 if nwords > 0 or p > max_: 795 self._error = 'incomplete reply' 796 return None 797 return res 798 799 def Status(self): 800 sock = self._Connect() 801 if not sock: 802 return None 803 req = pack('>2HLL', SEARCHD_COMMAND_STATUS, VER_COMMAND_STATUS, 4, 1) 804 self._Send(sock, req) 805 response = self._GetResponse(sock, VER_COMMAND_STATUS) 806 if not response: 807 return None 808 res = [] 809 p = 8 810 max_ = len(response) 811 while p < max_: 812 length = unpack('>L', response[p:p + 4])[0] 813 k = response[p + 4:p + length + 4] 814 p += 4 + length 815 length = unpack('>L', response[p:p + 4])[0] 816 v = response[p + 4:p + length + 4] 817 p += 4 + length 818 res += [[k, v]] 819 820 return res 821 822 def Open(self): 823 if self._socket: 824 self._error = 'already connected' 825 return None 826 server = self._Connect() 827 if not server: 828 return None 829 request = pack('>hhII', SEARCHD_COMMAND_PERSIST, 0, 4, 1) 830 self._Send(server, request) 831 self._socket = server 832 return True 833 834 def Close(self): 835 if not self._socket: 836 self._error = 'not connected' 837 return 838 self._socket.close() 839 self._socket = None 840 841 def EscapeString(self, string): 842 return re.sub('([=\\(\\)|\\-!@~\\"&/\\\\\\^\\$\\=\\<])', '\\\\\\1', string) 843 844 def FlushAttributes(self): 845 sock = self._Connect() 846 if not sock: 847 return -1 848 request = pack('>hhI', SEARCHD_COMMAND_FLUSHATTRS, VER_COMMAND_FLUSHATTRS, 0) 849 self._Send(sock, request) 850 response = self._GetResponse(sock, VER_COMMAND_FLUSHATTRS) 851 if not response or len(response) != 4: 852 self._error = 'unexpected response length' 853 return -1 854 tag = unpack('>L', response[0:4])[0] 855 return tag 856 857 858 def AssertInt32(value): 859 pass 860 861 862 def AssertUInt32(value): 863 pass 864 865 866 def SetBit(flag, bit, on): 867 if on: 868 flag += 1 << bit 869 else: 870 reset = 255 ^ 1 << bit 871 flag = flag & reset 872 return flag