|
| 1 | +import cPickle |
| 2 | +import collections |
| 3 | +import json |
| 4 | +import socket |
| 5 | +import struct |
| 6 | + |
| 7 | +from utils import query_repr_to_sql_query |
| 8 | + |
| 9 | + |
| 10 | +QueryPerformanceSubset = collections.namedtuple( |
| 11 | + "QueryPerformanceSubset", |
| 12 | + ["num_ret", "exe_time"] |
| 13 | +) |
| 14 | +QueryPerformance = collections.namedtuple( |
| 15 | + "QueryPerformance", |
| 16 | + ["num_ret", "num_rel", "num_rel_ret", "exe_time"] |
| 17 | +) |
| 18 | + |
| 19 | + |
| 20 | +class SocketChannel(object): |
| 21 | + _length_format = "<I" |
| 22 | + _length_size = struct.calcsize(_length_format) |
| 23 | + |
| 24 | + def __init__(self, host, port): |
| 25 | + self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
| 26 | + self._sock.connect((host, port)) |
| 27 | + |
| 28 | + def close(self): |
| 29 | + self._sock.close() |
| 30 | + |
| 31 | + def send_request(self, request): |
| 32 | + assert isinstance(request, dict) |
| 33 | + self._send_msg(json.dumps(request)) |
| 34 | + |
| 35 | + def receive_reply(self): |
| 36 | + return json.loads(self._recv_msg()) |
| 37 | + |
| 38 | + def _recvall(self, n): |
| 39 | + # Helper function to recv n bytes or return None if EOF is hit |
| 40 | + data = b'' |
| 41 | + while len(data) < n: |
| 42 | + packet = self._sock.recv(n - len(data)) |
| 43 | + if not packet: |
| 44 | + return None |
| 45 | + data += packet |
| 46 | + return data |
| 47 | + |
| 48 | + def _send_msg(self, msg): |
| 49 | + # Prefix each message with a 4-byte length (network byte order) |
| 50 | + msg = struct.pack(SocketChannel._length_format, len(msg)) + msg |
| 51 | + self._sock.sendall(msg) |
| 52 | + |
| 53 | + def _recv_msg(self): |
| 54 | + # Read message length and unpack it into an integer |
| 55 | + raw_msglen = self._recvall(SocketChannel._length_size) |
| 56 | + if not raw_msglen: |
| 57 | + return None |
| 58 | + msglen = struct.unpack(SocketChannel._length_format, raw_msglen)[0] |
| 59 | + # Read the message data |
| 60 | + return self._recvall(msglen) |
| 61 | + |
| 62 | + |
| 63 | +class IndexCursor(object): |
| 64 | + def __init__(self, index_cache, db_cursor): |
| 65 | + self._index_cache = index_cache |
| 66 | + self._db_cursor = db_cursor |
| 67 | + |
| 68 | + def close(self): |
| 69 | + self._db_cursor.close() |
| 70 | + |
| 71 | + def __enter__(self): |
| 72 | + return self |
| 73 | + |
| 74 | + def __exit__(self, *exc_info): |
| 75 | + self.close() |
| 76 | + |
| 77 | + def get_performance(self, query_repr, document_id_list=None, document_id_list_key=None, |
| 78 | + include_time=True, force=False): |
| 79 | + # check parameters |
| 80 | + assert isinstance(query_repr, (list, tuple)) |
| 81 | + assert document_id_list is None or (isinstance(document_id_list, (list, tuple)) and len(document_id_list) > 0 and all(isinstance(doc_id, (int, long)) for doc_id in document_id_list)) |
| 82 | + assert document_id_list_key is None or isinstance(document_id_list_key, (int, long)) |
| 83 | + assert (document_id_list_key is None) == (document_id_list is None) |
| 84 | + assert isinstance(include_time, bool) |
| 85 | + assert isinstance(force, bool) |
| 86 | + |
| 87 | + zero_document_id_list = (document_id_list is None) or (len(document_id_list) == 0) |
| 88 | + |
| 89 | + # transform the query representation in a query string |
| 90 | + sql_str = query_repr_to_sql_query(query_repr) |
| 91 | + |
| 92 | + # get the entry from the cache |
| 93 | + key = sql_str if zero_document_id_list else (sql_str, document_id_list_key) |
| 94 | + if not force: |
| 95 | + query_performance = self._index_cache._get(key) |
| 96 | + if query_performance is not None and (not include_time or query_performance.exe_time is not None): |
| 97 | + return query_performance |
| 98 | + |
| 99 | + # transform the document_id_list |
| 100 | + document_id_list = [] if document_id_list is None else list(set(document_id_list)) |
| 101 | + |
| 102 | + request = { |
| 103 | + "query": sql_str, |
| 104 | + "query_type": "cnf" |
| 105 | + } |
| 106 | + if not zero_document_id_list: |
| 107 | + request["rel"] = document_id_list |
| 108 | + self._db_cursor.send_request(request) |
| 109 | + |
| 110 | + result = self._db_cursor.receive_reply() |
| 111 | + if "error" in result: |
| 112 | + raise Exception(result["error"]) |
| 113 | + |
| 114 | + # compose the resulting object |
| 115 | + if zero_document_id_list: |
| 116 | + query_performance = QueryPerformanceSubset( |
| 117 | + num_ret=int(result["num_ret"]), |
| 118 | + exe_time=float(result["exe_time"]) |
| 119 | + ) |
| 120 | + else: |
| 121 | + query_performance = QueryPerformance( |
| 122 | + num_ret=int(result["num_ret"]), |
| 123 | + num_rel=int(result["num_rel"]), |
| 124 | + num_rel_ret=int(result["num_rel_ret"]), |
| 125 | + exe_time=float(result["exe_time"]) |
| 126 | + ) |
| 127 | + |
| 128 | + # put the result into the cache |
| 129 | + self._index_cache._put(key, query_performance) |
| 130 | + if not zero_document_id_list: |
| 131 | + qps = self._index_cache._get(key[0]) |
| 132 | + if qps is None or (include_time and qps.exe_time is None): |
| 133 | + qps = QueryPerformanceSubset( |
| 134 | + num_ret=query_performance.num_ret, |
| 135 | + exe_time=query_performance.exe_time |
| 136 | + ) |
| 137 | + self._index_cache._put(key[0], qps) |
| 138 | + |
| 139 | + # return |
| 140 | + return query_performance |
| 141 | + |
| 142 | + |
| 143 | +class IndexCache(object): |
| 144 | + def __init__(self, host, port): |
| 145 | + assert isinstance(host, str) |
| 146 | + assert isinstance(port, int) |
| 147 | + |
| 148 | + self._host = host |
| 149 | + self._port = port |
| 150 | + self._cache = dict() |
| 151 | + |
| 152 | + @staticmethod |
| 153 | + def load(file_path): |
| 154 | + host, port, cache = cPickle.load(open(file_path, "rb")) |
| 155 | + index_cache = IndexCache(host, port) |
| 156 | + index_cache._cache = cache |
| 157 | + return index_cache |
| 158 | + |
| 159 | + def dump(self, file_path): |
| 160 | + cPickle.dump( |
| 161 | + (self._host, self._port, self._cache), |
| 162 | + open(file_path, "wb"), |
| 163 | + protocol=cPickle.HIGHEST_PROTOCOL |
| 164 | + ) |
| 165 | + |
| 166 | + def __len__(self): |
| 167 | + return len(self._cache) |
| 168 | + |
| 169 | + def _get(self, key): |
| 170 | + return self._cache.get(key, None) |
| 171 | + |
| 172 | + def _put(self, key, value): |
| 173 | + self._cache[key] = value |
| 174 | + |
| 175 | + def cursor(self): |
| 176 | + connection = SocketChannel(host=self._host, port=self._port) |
| 177 | + return IndexCursor(self, connection) |
0 commit comments