Swarm-NG  1.1
logdb.py
Go to the documentation of this file.
1 ## @file logdb.py Routines and classes to access the BDB log file
2 # To read the documentation generated from this file refer to @ref swarmng.logdb
3 
4 ## @package swarmng.logdb
5 # Interface to the Log files generated by bdb_writer plugin
6 
7 from bsddb3.db import *
8 from logrecord import LogRecord
9 from struct import pack, unpack
10 import sys
11 import os
12 
13 TIME_EPSILON = sys.float_info.epsilon
14 
15 ## Primary key of the log file database
16 #
17 # Most of methods are only for internal use and are not documented.
18 class PKey:
19  ## time field of the associated LogRecord, a floating point value representing time in AU
20  time = property
21  ## Event ID of the associated LogRecord, integer value that is usually 1.
22  event_id = property
23  ## Integer ID of the system that the associated LogRecord has a snapshot.
24  system_id = property
25 
26  ## Create a primary key object from time, event_id and system_id respectively.
27  def __init__(self, time, event_id, system_id):
28  self.time = time
29  self.event_id = event_id
30  self.system_id = system_id
31 
32  @staticmethod
33  def decomposeBinary(bin):
34  if(len(bin) == 8):
35  t, se = unpack('fI', bin)
36  s = se % 2**24
37  e = se / 2**24
38  return (t,e,s)
39  else:
40  return None
41 
42  @staticmethod
43  def fromBinary(bin):
44  (t,e,s) = PKey.decomposeBinary(bin)
45  return PKey(t,e,s)
46 
47  @staticmethod
48  def compareBinary(s1,s2):
49  d1 = PKey.decomposeBinary(s1)
50  d2 = PKey.decomposeBinary(s2)
51  if(d1 == d2):
52  return 0
53  elif (d1 < d2):
54  return -1
55  else:
56  return 1
57 
58  def __repr__(self):
59  m = {'time':self.time, 'sys': self.system_id, 'evt': self.event_id }
60  return m.__repr__();
61 
62  def toBinary(self):
63  return pack('fI', self.time, self.event_id * 2**24 + self.system_id)
64 
65  @staticmethod
66  def packSys(s):
67  return pack('I',s)
68  @staticmethod
69  def packTime(t):
70  return pack('f',t)
71  @staticmethod
72  def packEvt(e):
73  return pack('B',s)
74  @staticmethod
75  def unpackSys(bin):
76  return unpack('I',bin)[0]
77  @staticmethod
78  def unpackEvt(bin):
79  return unpack('B',bin)[0]
80  @staticmethod
81  def unpackTime(bin):
82  return unpack('f',bin)[0]
83 
84 
85 
86 
87 def extract_sys(key, data):
88  return PKey.fromBinary(key).system_id
89 
90 def extract_time(key, data):
91  return PKey.fromBinary(key).time
92 
93 def extract_evt(key, data):
94  return PKey.fromBinary(key).event_id
95 
96 def compare_time(l, r):
97  if(len(l) < len(r)):
98  return -1
99  if(len(l) > len(r)):
100  return 1
101  elif(len(l) != 4):
102  return 0
103 
104  lf = PKey.unpackTime(l)
105  rf = PKey.unpackTime(r)
106  if( lf < rf ):
107  return -1
108  if( lf > rf ):
109  return 1
110  else:
111  return 0
112 
113 
114 def compare_sys(l, r):
115  if(len(l) < len(r)):
116  return -1
117  if(len(l) > len(r)):
118  return 1
119  elif(len(l) != 4):
120  return 0
121 
122  lf = PKey.unpackSys(l)
123  rf = PKey.unpackSys(r)
124  if( lf < rf ):
125  return -1
126  if( lf > rf ):
127  return 1
128  else:
129  return 0
130 
131 def compare_evt(l, r):
132  if(len(l) < len(r)):
133  return -1
134  if(len(l) > len(r)):
135  return 1
136  elif(len(l) != 1):
137  return 0
138 
139  lf = PKey.unpackEvt(l)
140  rf = PKey.unpackEvt(r)
141  if( lf < rf ):
142  return -1
143  if( lf > rf ):
144  return 1
145  else:
146  return 0
147 
148 def iter_cursor(c, mode = DB_NEXT):
149  while True:
150  n = c.get(mode)
151  if n != None :
152  yield n
153  else:
154  c.close()
155  raise StopIteration
156 
157 def iter_secondary_cursor(c, mode = DB_NEXT):
158  while True:
159  n = c.pget(mode)
160  if n != None :
161  yield n
162  else:
163  c.close()
164  raise StopIteration
165 
166 ## Interface to BDB log files. open a log file like:
167 #
168 # @code{.py}
169 # >>> db = IndexedLogDB('mydatabase.db')
170 # >>> records = db.all_records()
171 # >>> print(next(records))
172 # >>> print(next(records))
173 # @endcode
174 #
175 # Note that most of the methods below return key-value pairs. The
176 # key-value pair is a Python tuple where the first element is of type \ref PKey
177 # and has time, system_id, event_id properties. the second element is a \ref swarmng.logrecord.LogRecord "LogRecord"
178 # object and contains all the information about the planetary system and its attributes.
179 #
181  ## Supported version of BDB log files
182  #
183  fileFormatVersion = "1"
184 
185 
186  ## Opens a BDB file that contains the primary database and
187  # secondary indices.
188  # @arg @c fn: path to the database file name
189  def __init__(self, pathName):
190  CACHESIZE = 1024*1024*64 ;
191  e = DBEnv()
192  e.set_cachesize(0,CACHESIZE,0)
193  #print("Trying to initialize environment for `{0}`".format(os.path.dirname(pathName)))
194  e.open(os.path.dirname(pathName),DB_INIT_CDB | DB_INIT_MPOOL)
195  fn = os.path.basename(pathName);
196  #print("Opening `{0}`".format(fn))
197 
198  m = DB(e)
199  m.open(fn, dbname="metadata", flags=DB_RDONLY)
200 
201  p = DB(e)
202  p.set_bt_compare(PKey.compareBinary)
203  p.open(fn, dbname="primary", flags=DB_RDONLY)
204 
205  si = DB(e)
206  si.set_bt_compare(compare_sys)
207  si.set_dup_compare(PKey.compareBinary)
208  si.open(fn, dbname="system_idx", flags=DB_RDONLY)
209 
210  ti = DB(e)
211  ti.set_bt_compare(compare_time)
212  ti.set_dup_compare(PKey.compareBinary)
213  ti.open(fn, dbname="time_idx", flags=DB_RDONLY)
214 
215  ei = DB(e)
216  ei.set_bt_compare(compare_evt)
217  ei.set_dup_compare(PKey.compareBinary)
218  ei.open(fn, dbname="event_idx", flags=DB_RDONLY)
219 
220  p.associate(si, extract_sys )
221  p.associate(ti, extract_time )
222  p.associate(ei, extract_evt )
223 
224  self.primary = p
225  self.system_idx = si
226  self.time_idx = ti
227  self.event_idx = ei
228  self.metadata = m
229 
230  self.validateVersionInfo()
231 
232  ## Get meta data from the database for the provided `name` string.
233  #
234  # There are two metadata that are always available: `fileFormatVersion` and `swarmngVersion`
235  #
236  # Meta data for the log file is a mapping of string -> string.
237  #
238  #
239  # @arg \c name : string :name of the property we are looking for.
240  # returns : string value for the provided property, None if the property is not found.
241  def getMetadata(self,name):
242  return self.metadata.get(name)
243 
244  def validateVersionInfo(self):
245  v = self.getMetadata("fileFormatVersion")
246  if v != self.fileFormatVersion :
247  raise RuntimeError("Mismatching file format version: {0}, required {1}".format(v, self.fileFormatVersion))
248 
249 
250  ## Return an iterable for all of the records, the records are
251  # sorted by time, then event id then system id.
252  #
253  # The elements of the returend iterable or key-value pairs.
254  #
255  def all_records(self):
256  c = self.primary.cursor()
257  for k,l in iter_cursor(c):
258  yield IndexedLogDB.decodeKVP((k,l))
259 
260  ## Returns a iterable of times for a time range, it is used for making
261  # range queries.
262  #
263  # The elements of the returned iterable are floating point values.
264  def time_sequence(self, time_range):
265 
266  t0, t1 = time_range
267  c = self.time_idx.cursor()
268  k = PKey.packTime(t0)
269  c.set_range(k)
270  c.prev();
271  for k, p, l in iter_secondary_cursor(c, DB_NEXT_NODUP):
272  t = PKey.unpackTime(k)
273  if t <= t1 :
274  yield t
275  else:
276  raise StopIteration
277  ## Query for all the records that have system id in the system range
278  #
279  # Return an iterable of key-value pairs.
280  def system_range_records(self, sys_range):
281  s0, s1 = sys_range
282  c = self.system_idx.cursor()
283  k = PKey.packSys(s0)
284  c.set_range(k)
285  c.prev()
286  for k, p, l in iter_secondary_cursor(c):
287  s = PKey.unpackSys(k)
288  if s <= s1 :
289  yield IndexedLogDB.decodeKVP((p,l))
290  else:
291  raise StopIteration
292  ## Query for all the records that have time in the time range
293  #
294  # Return an iterable of key-value pairs.
295  def time_range_records(self, time_range):
296  t0, t1 = time_range
297  c = self.time_idx.cursor()
298  k = PKey.packTime(t0)
299  c.set_range(k)
300  c.prev()
301  for k, p, l in iter_secondary_cursor(c):
302  t = PKey.unpackTime(k)
303  if t <= t1 :
304  yield IndexedLogDB.decodeKVP((p,l))
305  else:
306  raise StopIteration
307 
308  ## Query for all the records for a system in a given time range.
309  #
310  # Return an iterable of key-value pairs.
311  def system_at_time(self, sysid, time_range):
312  t0, t1 = time_range
313  c = self.system_idx.cursor()
314  k = PKey.packSys(sysid)
315  pk = PKey(t0, 1, sysid).toBinary()
316  _, pk, _ = c.pget(k, pk, DB_GET_BOTH_RANGE)
317  c.prev()
318  for k, pk, l in iter_secondary_cursor(c):
319  kk = PKey.fromBinary(pk)
320  if t0 <= kk.time <= t1 and kk.system_id == sysid :
321  yield IndexedLogDB.decodeKVP((pk,l))
322  else:
323  raise StopIteration
324 
325  ## Return initial coniditions for the system range
326  #
327  # Return an iterable of a tuple of system id and LogRecord.
328  #
329  # Usage:
330  # @code{.py}
331  # for system_id, lr in d.initial_conditions(Range.interval(10,20)):
332  # print(system_id) # prints integers
333  # print(lr.time) # lr is a LogRecord object and has a time property
334  # @endcode
335  #
336  def initial_conditions(self, system_range):
337  c = self.system_idx.cursor()
338 
339  if system_range.isUniversal() :
340  s0 = 0
341  s1 = sys.maxint
342  r = c.first()
343  else:
344  s0, s1 = system_range.ulPair()
345  k = PKey.packSys(s0)
346  r = c.set_range(k)
347 
348  sysid = s0
349  while sysid < s1 :
350  if r :
351  ks, l = r
352  sysid = PKey.unpackSys(ks)
353  yield sysid, LogRecord.from_binary(l)
354  else:
355  break
356  r = c.get(DB_NEXT_NODUP)
357  ## Return the final conditions for a range of systems that
358  # is the largest time for which the system has a valid entry
359  #
360  # Return an iterable of a tuple of system id and LogRecord.
361  #
362  # Usage:
363  # @code{.py}
364  # for system_id, lr in d.final_conditions(Range.interval(10,20)):
365  # print(system_id) # prints integers
366  # print(lr.time) # lr is a LogRecord object and has a time property
367  # @endcode
368  #
369  def final_conditions(self,system_range):
370  c = self.system_idx.cursor()
371 
372  if system_range.isUniversal() :
373  s0 = 0
374  s1 = sys.maxint
375  c.first()
376  else:
377  s0, s1 = system_range.ulPair()
378  k = PKey.packSys(s0)
379  c.set_range(k)
380 
381  sysid = s0-1
382  while sysid < s1 and c.next() != None :
383  if c.get(DB_NEXT_NODUP) == None :
384  ks, l = c.last()
385  else:
386  ks, l = c.prev()
387 
388  sysid = PKey.unpackSys(ks)
389  yield sysid, LogRecord.from_binary(l)
390 
391 
392 
393  @staticmethod
394  def decodeKVP(r):
395  return (PKey.fromBinary(r[0]),LogRecord.from_binary(r[1]))
396 
397  def system_range_for_time_event(self,time,event_id,system_range):
398  c = self.primary.cursor()
399  s0, s1 = system_range
400 
401  k = PKey(time, event_id, s0)
402  c.set_range(k.toBinary())
403  # We have to check that we are at a valid location
404  c.prev()
405  for r in iter_cursor(c):
406  k,l = IndexedLogDB.decodeKVP(r)
407  if s0 <= k.system_id <= s1 and k.time == time and k.event_id == event_id:
408  yield (k,l)
409  else:
410  raise StopIteration
411 
412  ## Query the database for with a time range ,system range and event id range
413  #
414  # @arg tr: time range (of type swarmng.range_type.Range)
415  # @arg sr: system range (of type swarmng.range_type.Range)
416  # @arg er: event ID range (of type swarmng.range_type.Range)
417  #
418  # Return an iterable of key-value pairs.
419  def query(d , tr, sr, er ):
420 
421  def filterEventID(q,er):
422  for k,l in q:
423  if(er.contains(k.event_id)):
424  yield (k,l)
425 
426  q0 = d.queryInternal(tr,sr)
427 
428  if(not er.isUniversal()):
429  q = filterEventID(q0,er)
430  else:
431  q = q0
432 
433  return q
434 
435  def queryInternal(d, tr, sr):
436  if(tr.isUniversal()):
437  if(sr.isUniversal()):
438  for k,l in d.all_records():
439  yield (k,l)
440  else:
441  for k,l in d.system_range_records(sr.ulPair()):
442  yield (k,l)
443  else:
444  if(sr.isUniversal()):
445  for k,l in d.time_range_records(tr.ulPair()):
446  yield (k,l)
447  else:
448  for t in d.time_sequence(tr.ulPair()):
449  for k,l in d.system_range_for_time_event(t,1,sr.ulPair()):
450  yield (k,l)
451 
452 
453