part.h
Go to the documentation of this file.
1 // File: $Id$
2 // Author: John Wu <John.Wu at ACM.org>
3 // Copyright (c) 2000-2016 the Regents of the University of California
4 #ifndef IBIS_PART_H
5 #define IBIS_PART_H
6 #include "column.h"
13 #include "resource.h"
14 #include "utilidor.h"
15 
16 #include <string>
17 #include <vector>
18 
27 class FASTBIT_CXX_DLLSPEC ibis::part {
28 public:
29  enum TABLE_STATE {
30  UNKNOWN_STATE=0, STABLE_STATE, RECEIVING_STATE,
31  PRETRANSITION_STATE, TRANSITION_STATE, POSTTRANSITION_STATE
32  };
33  struct info; // To hold some basic information about a table.
34  class readLock;
35 
36  /******************************************************************/
37  // public functions
38  /******************************************************************/
39 
41  virtual ~part();
43  explicit part(const char* name=0, bool ro=false);
45  part(const char* adir, const char* bdir, bool ro=false);
47  part(const std::vector<const char*> &mtags, bool ro=false);
49  part(const ibis::resource::vList &mtags, bool ro=false);
50 
51  inline info* getInfo() const;
52  TABLE_STATE getState() const;
54  TABLE_STATE getStateNoLocking() const {return state;}
55 
56  virtual int buildIndexes(const char* iopt, int nthr);
57  virtual int buildIndexes(const ibis::table::stringArray&, int nthr=1);
58  void buildSorted(const char* colname) const;
59  void loadIndexes(const char* iopt=0, int ropt=0) const;
60  void unloadIndexes() const;
61  void purgeIndexFiles() const;
62 
64  const char* name() const {return (m_name?m_name:"?");}
66  const char* description() const {return m_desc.c_str();}
68  const char* indexSpec() const {return idxstr;}
70  void indexSpec(const char*);
72  time_t timestamp() const {return switchTime;}
75  std::string metaTags() const;
76  ibis::table::stringArray columnNames() const;
77  ibis::table::typeArray columnTypes() const;
78  column* getColumn(const char* name) const;
79  column* getColumn(uint32_t ind) const;
80 
82  const char* currentDataDir() const {return activeDir;}
84  uint32_t nColumns() const {return columns.size();}
86  uint32_t nRows() const {return nEvents;}
87 
89  void print(std::ostream &out) const;
90  void logWarning(const char* event, const char* fmt, ...) const;
91  void logMessage(const char* event, const char* fmt, ...) const;
92 
94  bool matchNameValuePair(const char* name, const char* value) const;
97  bool matchMetaTags(const std::vector<const char*> &mtags) const;
99  bool matchMetaTags(const ibis::resource::vList &mtags) const;
101  inline const char* getMetaTag(const char*) const;
102 
104  virtual long selfTest(int nth=1, const char* pref=0) const;
105  void computeMinMax();
106 
107  /******************************************************************/
108  // The following functions need to hold a read lock on the partition,
109  // however in order for them to produce consistent results, they must
110  // share the same read lock. These functions do not contain the lock
111  // themselves, but rely on the caller to maintain a consistent lock.
112  /******************************************************************/
114  uint32_t getRowNumber(const rid_t &rid) const;
115  long evaluateRIDSet(const ibis::RIDSet&, ibis::bitvector&) const;
116  array_t<rid_t>* getRIDs() const {return rids;} // all RIDs
117  array_t<rid_t>* getRIDs(const ibis::bitvector &mask) const;// some RIDs
120  bool explicitRIDs() const {
121  return (rids!=0) ? (rids->size()==nEvents) : false;}
122 
123  virtual double estimateCost(const ibis::qContinuousRange &cmp) const;
124  virtual double estimateCost(const ibis::qDiscreteRange &cmp) const;
125  virtual double estimateCost(const ibis::qIntHod &cmp) const;
126  virtual double estimateCost(const ibis::qUIntHod &cmp) const;
127  virtual double estimateCost(const ibis::qString &cmp) const;
128  virtual double estimateCost(const ibis::qAnyString &cmp) const;
129 
131  virtual long estimateRange(const ibis::qContinuousRange &cmp) const;
132 
134  virtual long estimateRange(const ibis::qDiscreteRange &cmp) const;
136  virtual long estimateRange(const ibis::qIntHod &cmp) const;
138  virtual long estimateRange(const ibis::qUIntHod &cmp) const;
139 
141  virtual long evaluateRange(const ibis::qContinuousRange &cmp,
142  const ibis::bitvector &mask,
143  ibis::bitvector &res) const;
145  virtual long evaluateRange(const ibis::qDiscreteRange &cmp,
146  const ibis::bitvector &mask,
147  ibis::bitvector &res) const;
149  virtual long evaluateRange(const ibis::qIntHod &cmp,
150  const ibis::bitvector &mask,
151  ibis::bitvector &res) const;
153  virtual long evaluateRange(const ibis::qUIntHod &cmp,
154  const ibis::bitvector &mask,
155  ibis::bitvector &res) const;
156 
158  virtual long estimateRange(const ibis::qContinuousRange &cmp,
159  ibis::bitvector &low,
160  ibis::bitvector &high) const;
161 
163  virtual float getUndecidable(const ibis::qContinuousRange &cmp,
164  ibis::bitvector &iffy) const;
165 
167  virtual long estimateRange(const ibis::qDiscreteRange &cmp,
168  ibis::bitvector &low,
169  ibis::bitvector &high) const;
171  virtual float getUndecidable(const ibis::qDiscreteRange &cmp,
172  ibis::bitvector &iffy) const;
173 
175  virtual long estimateRange(const ibis::qIntHod &cmp,
176  ibis::bitvector &low,
177  ibis::bitvector &high) const;
179  virtual float getUndecidable(const ibis::qIntHod &cmp,
180  ibis::bitvector &iffy) const;
181 
183  virtual long estimateRange(const ibis::qUIntHod &cmp,
184  ibis::bitvector &low,
185  ibis::bitvector &high) const;
187  virtual float getUndecidable(const ibis::qUIntHod &cmp,
188  ibis::bitvector &iffy) const;
189 
190  virtual long doScan(const ibis::qRange &cmp,
191  ibis::bitvector &hits) const;
192  virtual long doScan(const ibis::qRange &cmp,
193  const ibis::bitvector &mask,
194  ibis::bitvector &hits) const;
195  virtual long doScan(const ibis::qRange &cmp,
196  const ibis::bitvector &mask,
197  void *res) const;
198  virtual long doScan(const ibis::qRange &cmp,
199  const ibis::bitvector &mask,
200  void *res, ibis::bitvector &hits) const;
201 
202  virtual long negativeScan(const ibis::qRange &cmp,
203  const ibis::bitvector &mask,
204  ibis::bitvector &hits) const;
205 
206  virtual long doScan(const ibis::math::term&, const ibis::bitvector&,
207  ibis::bitvector&) const;
208  virtual long doScan(const ibis::compRange &cmp,
209  ibis::bitvector &hits) const;
210  virtual long doScan(const ibis::compRange &cmp,
211  const ibis::bitvector &mask,
212  ibis::bitvector &hits) const;
213 
214  template <typename E> static long
215  doScan(const array_t<E> &varr,
216  const ibis::qRange &cmp,
217  const ibis::bitvector &mask,
218  ibis::bitvector &hits);
219  template <typename E> static long
220  doScan(const array_t<E> &varr,
221  const ibis::qContinuousRange &cmp,
222  const ibis::bitvector &mask,
223  ibis::bitvector &hits);
224  template <typename E> static long
225  doScan(const array_t<E> &varr,
226  const ibis::qContinuousRange &cmp,
227  const ibis::bitvector &mask,
228  array_t<E> &res);
229  template <typename E> static long
230  doScan(const array_t<E> &varr,
231  const ibis::qContinuousRange &cmp,
232  const ibis::bitvector &mask,
233  array_t<E> &res, ibis::bitvector &hits);
234 
235  long countHits(const ibis::qRange &cmp) const;
236 
237  virtual long estimateMatchAny(const ibis::qAnyAny &cmp,
238  ibis::bitvector &low,
239  ibis::bitvector &high) const;
240  virtual long matchAny(const ibis::qAnyAny &cmp,
241  ibis::bitvector &hits) const;
242  virtual long matchAny(const ibis::qAnyAny &cmp,
243  const ibis::bitvector &mask,
244  ibis::bitvector &hits) const;
245 
246  long patternSearch(const ibis::qLike &cmp,
247  ibis::bitvector &low) const;
248  long stringSearch(const ibis::qString &cmp,
249  ibis::bitvector &low) const;
250  long stringSearch(const ibis::qAnyString &cmp,
251  ibis::bitvector &low) const;
252  long keywordSearch(const ibis::qKeyword &cmp,
253  ibis::bitvector &low) const;
254  long keywordSearch(const ibis::qAllWords &cmp,
255  ibis::bitvector &low) const;
256  long patternSearch(const ibis::qLike &cmp) const;
257  long stringSearch(const ibis::qString &cmp) const;
258  long stringSearch(const ibis::qAnyString &cmp) const;
259  long keywordSearch(const ibis::qKeyword &cmp) const;
260  long keywordSearch(const ibis::qAllWords &cmp) const;
261 
266  int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
267  const ibis::bitvector &mask,
268  ibis::bitvector64 &pairs) const;
271  int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
272  const ibis::bitvector &mask,
273  const char* pairfile) const;
275  int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
276  const ibis::bitvector &mask) const;
279  int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
280  const ibis::bitvector &mask,
281  ibis::bitvector64 &pairs) const;
282  int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
283  const ibis::bitvector &mask) const;
288  int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
289  const ibis::bitvector64 &trial,
290  ibis::bitvector64 &result) const;
291  int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
292  const ibis::bitvector64 &trial,
293  ibis::bitvector64 &result) const;
294  /******************************************************************/
295 
297  array_t<signed char>*
298  selectBytes(const char* name, const ibis::bitvector &mask) const;
300  array_t<unsigned char>*
301  selectUBytes(const char* name, const ibis::bitvector &mask) const;
303  array_t<int16_t>*
304  selectShorts(const char* name, const ibis::bitvector &mask) const;
306  array_t<uint16_t>*
307  selectUShorts(const char* name, const ibis::bitvector &mask) const;
309  array_t<int32_t>*
310  selectInts(const char* name, const ibis::bitvector &mask) const;
312  array_t<uint32_t>*
313  selectUInts(const char* name, const ibis::bitvector &mask) const;
315  array_t<int64_t>*
316  selectLongs(const char* name, const ibis::bitvector &mask) const;
318  array_t<uint64_t>*
319  selectULongs(const char* name, const ibis::bitvector &mask) const;
321  array_t<float>*
322  selectFloats(const char* name, const ibis::bitvector &mask) const;
324  array_t<double>*
325  selectDoubles(const char* name, const ibis::bitvector &mask) const;
327  std::vector<std::string>*
328  selectStrings(const char* name, const ibis::bitvector &mask) const;
329 
330  long selectValues(const char* cname, const ibis::bitvector &mask,
331  void* vals) const;
332  long selectValues(const ibis::qContinuousRange& cond, void* vals) const;
333  long calculate(const ibis::math::term&, const ibis::bitvector&,
334  array_t<double>&) const;
335  long calculate(const ibis::math::stringFunction1&, const ibis::bitvector&,
336  std::vector<std::string>&) const;
337 
338  /******************************************************************/
339  // A group of functions added August, 2005 to provide summary
340  // information about the data. Logically, they are an extension of
341  // getInfo.
342 
344  double getActualMin(const char *name) const;
346  double getActualMax(const char *name) const;
348  double getColumnSum(const char *name) const;
349 
353  long get1DDistribution(const char *constraints, const char *cname,
354  double begin, double end, double stride,
355  std::vector<uint32_t> &counts) const;
357  long get2DDistribution(const char *constraints, const char *cname1,
358  double begin1, double end1, double stride1,
359  const char *cname2,
360  double begin2, double end2, double stride2,
361  std::vector<uint32_t> &counts) const;
363  long get3DDistribution(const char *constraints, const char *cname1,
364  double begin1, double end1, double stride1,
365  const char *cname2,
366  double begin2, double end2, double stride2,
367  const char *cname3,
368  double begin3, double end3, double stride3,
369  std::vector<uint32_t> &counts) const;
371  long get1DDistribution(const char *constraints, const char *cname,
372  double begin, double end, double stride,
373  const char *wtname,
374  std::vector<double> &weights) const;
376  long get2DDistribution(const char *constraints, const char *cname1,
377  double begin1, double end1, double stride1,
378  const char *cname2,
379  double begin2, double end2, double stride2,
380  const char *wtname,
381  std::vector<double> &weights) const;
383  long get3DDistribution(const char *constraints, const char *cname1,
384  double begin1, double end1, double stride1,
385  const char *cname2,
386  double begin2, double end2, double stride2,
387  const char *cname3,
388  double begin3, double end3, double stride3,
389  const char *wtname,
390  std::vector<double> &weights) const;
392  long get1DDistribution(const char *cname, uint32_t nbin,
393  std::vector<double> &bounds,
394  std::vector<uint32_t> &counts) const;
396  long get1DDistribution(const char *constraints,
397  const char *cname, uint32_t nbin,
398  std::vector<double> &bounds,
399  std::vector<uint32_t> &counts) const;
401  long get2DDistribution(const char *cname1, const char *cname2,
402  uint32_t nb1, uint32_t nb2,
403  std::vector<double> &bounds1,
404  std::vector<double> &bounds2,
405  std::vector<uint32_t> &counts,
406  const char* const option=0) const;
408  long get2DDistribution(const char *constraints,
409  const char *name1, const char *name2,
410  uint32_t nb1, uint32_t nb2,
411  std::vector<double> &bounds1,
412  std::vector<double> &bounds2,
413  std::vector<uint32_t> &counts) const;
415  long get3DDistribution(const char *cname1, const char *cname2,
416  const char *cname3,
417  uint32_t nb1, uint32_t nb2, uint32_t nb3,
418  std::vector<double> &bounds1,
419  std::vector<double> &bounds2,
420  std::vector<double> &bounds3,
421  std::vector<uint32_t> &counts,
422  const char* const option=0) const;
424  long get3DDistribution(const char *constraints,
425  const char *cname1, const char *cname2,
426  const char *cname3,
427  uint32_t nb1, uint32_t nb2, uint32_t nb3,
428  std::vector<double> &bounds1,
429  std::vector<double> &bounds2,
430  std::vector<double> &bounds3,
431  std::vector<uint32_t> &counts) const;
433  long get1DBins(const char *constraints, const char *cname,
434  double begin, double end, double stride,
435  std::vector<ibis::bitvector> &bins) const;
437  long get1DBins(const char *constraints, const char *cname,
438  double begin, double end, double stride,
439  std::vector<ibis::bitvector*> &bins) const;
441  long get1DBins(const char *constraints, const char *cname,
442  double begin, double end, double stride,
443  const char *wtname,
444  std::vector<double> &weights,
445  std::vector<ibis::bitvector*> &bins) const;
447  long get2DBins(const char *constraints, const char *cname1,
448  double begin1, double end1, double stride1,
449  const char *cname2,
450  double begin2, double end2, double stride2,
451  std::vector<ibis::bitvector> &bins) const;
453  long get2DBins(const char *constraints, const char *cname1,
454  double begin1, double end1, double stride1,
455  const char *cname2,
456  double begin2, double end2, double stride2,
457  std::vector<ibis::bitvector*> &bins) const;
459  long get2DBins(const char *constraints, const char *cname1,
460  double begin1, double end1, double stride1,
461  const char *cname2,
462  double begin2, double end2, double stride2,
463  const char *wtname,
464  std::vector<double> &weights,
465  std::vector<ibis::bitvector*> &bins) const;
467  long get3DBins(const char *constraints, const char *cname1,
468  double begin1, double end1, double stride1,
469  const char *cname2,
470  double begin2, double end2, double stride2,
471  const char *cname3,
472  double begin3, double end3, double stride3,
473  std::vector<ibis::bitvector> &bins) const;
475  long get3DBins(const char *constraints, const char *cname1,
476  double begin1, double end1, double stride1,
477  const char *cname2,
478  double begin2, double end2, double stride2,
479  const char *cname3,
480  double begin3, double end3, double stride3,
481  std::vector<ibis::bitvector*> &bins) const;
483  long get3DBins(const char *constraints, const char *cname1,
484  double begin1, double end1, double stride1,
485  const char *cname2,
486  double begin2, double end2, double stride2,
487  const char *cname3,
488  double begin3, double end3, double stride3,
489  const char *wtname,
490  std::vector<double> &weights,
491  std::vector<ibis::bitvector*> &bins) const;
494  long get1DBins(const char *constraints, const char *cname1, uint32_t nb1,
495  std::vector<double> &bounds1,
496  std::vector<ibis::bitvector> &bins) const;
498  long get2DBins(const char *constraints,
499  const char *cname1, const char *cname2,
500  uint32_t nb1, uint32_t nb2,
501  std::vector<double> &bounds1,
502  std::vector<double> &bounds2,
503  std::vector<ibis::bitvector> &bins) const;
505  long get3DBins(const char *constraints,
506  const char *cname1, const char *cname2, const char *cname3,
507  uint32_t nb1, uint32_t nb2, uint32_t nb3,
508  std::vector<double> &bounds1,
509  std::vector<double> &bounds2,
510  std::vector<double> &bounds3,
511  std::vector<ibis::bitvector> &bins) const;
513 
518  long getDistribution(const char *name,
519  std::vector<double> &bounds,
520  std::vector<uint32_t> &counts) const;
522  long getDistribution(const char *constraints,
523  const char *name,
524  std::vector<double> &bounds,
525  std::vector<uint32_t> &counts) const;
528  long getDistribution(const char *name, uint32_t nbc,
529  double *bounds, uint32_t *counts) const;
532  long getDistribution(const char *name, const char *constraints,
533  uint32_t nbc, double *bounds,
534  uint32_t *counts) const;
535 
537  long getJointDistribution(const char *constraints,
538  const char *name1, const char *name2,
539  std::vector<double> &bounds1,
540  std::vector<double> &bounds2,
541  std::vector<uint32_t> &counts) const;
542 
544  long getCumulativeDistribution(const char *name,
545  std::vector<double> &bounds,
546  std::vector<uint32_t> &counts) const;
549  long getCumulativeDistribution(const char *constraints,
550  const char *name,
551  std::vector<double> &bounds,
552  std::vector<uint32_t> &counts) const;
555  long getCumulativeDistribution(const char *name, uint32_t nbc,
556  double *bounds, uint32_t *counts) const;
559  long getCumulativeDistribution(const char *constraints, const char *name,
560  uint32_t nbc, double *bounds,
561  uint32_t *counts) const;
563 
564  /******************************************************************/
575  const std::vector<uint32_t>& getMeshShape() const {return shapeSize;}
578  const std::vector<std::string>& getMeshDimensions() const {
579  return shapeName;}
580  void setMeshShape(const ibis::array_t<uint64_t>&);
581  void setMeshShape(const char *shape);
582  void updateMetaData() const;
584  void combineNames(ibis::table::namesTypes &metalist) const;
585 
588  int clear();
589  int updateData();
591  long append(const char* dir);
596  long commit(const char* dir);
599  long rollback();
600  long addColumn(const char* aexpr, const char* cname,
601  ibis::TYPE_T ctype=ibis::DOUBLE);
602  long addColumn(const ibis::math::term* xpr, ibis::bitvector& mask,
603  const char* cname, ibis::TYPE_T ctype=ibis::DOUBLE);
604 
605  /******************************************************************/
606  virtual long reorder();
607  virtual long reorder(const ibis::table::stringArray &names);
608  virtual long reorder(const ibis::table::stringArray &names,
609  const std::vector<bool> &directions);
610 
611  long deactivate(const std::vector<uint32_t> &rows);
612  long deactivate(const char* conds);
613  long reactivate(const std::vector<uint32_t> &rows);
614  long reactivate(const char* conds);
615  long purgeInactive();
616  void emptyCache() const;
618  void getNullMask(ibis::bitvector &m) const {m.copy(amask);}
620  const ibis::bitvector& getMaskRef() const {return amask;}
621 
622  static const char* skipPrefix(const char*);
624  static char* readMetaTags(const char* const dir);
626  static void genName(const std::vector<const char*> &mtags,
627  std::string &name);
629  static void genName(const ibis::resource::vList &mtags,
630  std::string &name);
631  void rename(const ibis::partAssoc& known);
632  void rename(const char*);
633 
635  static uint32_t countPages(const ibis::bitvector &mask,
636  unsigned elemsize=4);
638  ibis::fileManager::ACCESS_PREFERENCE
639  accessHint(const ibis::bitvector &mask, unsigned elemsize=4) const;
640 
642  struct thrArg {
643  const part* et;
644  const char* pref;
645  long* nerrors;
646  ibis::util::counter cnt;
647  std::vector<std::string> conds;
648  std::vector<unsigned> super;
649  std::vector<unsigned> hits;
650  };
652  struct indexBuilderPool {
653  ibis::util::counter cnt;
654  ibis::table::stringArray opt;
655  const part &tbl;
656  indexBuilderPool(const part &t, const char* spec)
657  : cnt(), tbl(t) {opt.push_back(spec);}
658  indexBuilderPool(const part&, const ibis::table::stringArray&);
659  };
660 
662  void queryTest(const char* pref, long* nerrors) const;
664  void quickTest(const char* pref, long* nerrors) const;
667  void testRangeOperators(const ibis::column* col,
668  long* nerrors) const;
669 
670  void doBackup();
671 
673  typedef std::map< const char*, column*, lessi > columnList;
674 
675  template <typename T> static int
676  writeColumn(int fdes, ibis::bitvector::word_t nold,
677  ibis::bitvector::word_t nnew,
678  ibis::bitvector::word_t voffset,
679  const array_t<T>& vals, const T& fill,
680  ibis::bitvector& totmask,
681  const ibis::bitvector& newmask);
682  static int writeStrings(const char *fnm, ibis::bitvector::word_t nold,
683  ibis::bitvector::word_t nnew,
684  ibis::bitvector::word_t voffset,
685  const std::vector<std::string>& vals,
686  ibis::bitvector& totmask,
687  const ibis::bitvector& newmask);
688  static int writeRaw(int bdes, int sdes, ibis::bitvector::word_t nold,
689  ibis::bitvector::word_t nnew,
690  ibis::bitvector::word_t voffset,
691  const ibis::array_t<unsigned char>& bytes,
692  const ibis::array_t<int64_t>& starts,
693  ibis::bitvector& totmask,
694  const ibis::bitvector& newmask);
695  static int writeOpaques(int bdes, int sdes,
696  ibis::bitvector::word_t nold,
697  ibis::bitvector::word_t nnew,
698  ibis::bitvector::word_t voffset,
699  const std::vector<ibis::opaque>& opq,
700  ibis::bitvector& totmask,
701  const ibis::bitvector& newmask);
702  int gainReadAccess() const;
703  int releaseAccess() const;
704  int gainWriteAccess() const;
705  int tryWriteAccess() const;
707  class barrel;
708  class vault;
709 
710 protected:
711  class cleaner;
712  class mutexLock;
713  class writeLock;
714  class softWriteLock;
715 
716  friend struct info;
717  friend class cleaner;
718  friend class readLock;
719  friend class writeLock;
720  friend class softWriteLock;
721  friend class mutexLock;
722 
723  /******************************************************************/
724  // protected member variables
725  //
726  char* m_name;
727  std::string m_desc;
728  ibis::resource::vList metaList;
729  mutable array_t<rid_t>* rids;
730  columnList columns;
731  uint32_t nEvents;
732  char* activeDir;
733  char* backupDir;
734  time_t switchTime;
735  TABLE_STATE state;
736  char* idxstr;
737 
738  ibis::bitvector amask;
739  std::vector<const column*> colorder;
740  std::vector<std::string> shapeName;
741  std::vector<uint32_t> shapeSize;
742 
743  ibis::part::cleaner* myCleaner;
744  bool readonly;
745 
746 
747  /******************************************************************/
748  // protected member functions
749  //
751  int readMetaData(uint32_t &nrows, columnList &plist, const char* dir);
753  void writeMetaData(const uint32_t nrows, const columnList &plist,
754  const char* dir) const;
755  void readRIDs() const;
756  void freeRIDs() const;
757 
758  // functions to deal with meta tags -- those attributes that are
759  // associated with a whole partition but are expected to be processed
760  // like a categorical attribute in queries
761  void extendMetaTags();
762  void setMetaTags(const ibis::resource::vList &mts);
763  void setMetaTags(const std::vector<const char*> &mts);
764 
766  void readMeshShape(const char* const dir);
768  void digestMeshShape(const char* shape);
769 
771  void logError(const char* event, const char* fmt, ...) const;
772 
773  void makeBackupCopy(); // copy the content of activeDir to backupDir
774  long verifyBackupDir(); // minimal consistency check
775  void deriveBackupDirName();
776  long appendToBackup(const char* dir); // append to the backup directory
777 
779  template <typename T>
780  long writeValues(const char *fname, const array_t<uint32_t> &ind);
782  template <typename T>
783  long reorderValues(const char *fname,
784  array_t<uint32_t> &starts,
785  array_t<uint32_t> &indout,
786  const array_t<uint32_t> &indin,
787  bool ascending);
788  long append1(const char* dir);
789  long append2(const char* dir);
790 
792  long deactivate(const ibis::bitvector &rows);
794  long reactivate(const ibis::bitvector &rows);
796  void numbersToBitvector(const std::vector<uint32_t>&,
797  ibis::bitvector&) const;
798  void stringToBitvector(const char*, ibis::bitvector&) const;
799 
800  template <typename T> static long
801  doCompare(const array_t<T> &array,
802  const ibis::qRange &cmp,
803  const ibis::bitvector &mask,
804  ibis::array_t<T> &res, ibis::bitvector &hits);
805  template <typename T> static long
806  doCompare(const char *file,
807  const ibis::qRange &cmp,
808  const ibis::bitvector &mask,
809  ibis::array_t<T> &res, ibis::bitvector &hits);
810 
811  template <typename T> static long
812  doCompare(const array_t<T> &array,
813  const ibis::qRange &cmp,
814  const ibis::bitvector &mask,
815  ibis::array_t<T> &res);
816  template <typename T> static long
817  doCompare(const char *file,
818  const ibis::qRange &cmp,
819  const ibis::bitvector &mask,
820  ibis::array_t<T> &res);
821 
822  template <typename T> static long
823  doCompare(const array_t<T> &array,
824  const ibis::qRange &cmp,
825  const ibis::bitvector &mask,
826  ibis::bitvector &hits);
827  template <typename T> static long
828  doCompare(const char *file,
829  const ibis::qRange &cmp,
830  const ibis::bitvector &mask,
831  ibis::bitvector &hits);
832 
833  template <typename T> static long
834  negativeCompare(const array_t<T> &array,
835  const ibis::qRange &cmp,
836  const ibis::bitvector &mask,
837  ibis::bitvector &hits);
838  template <typename T> static long
839  negativeCompare(const char *file,
840  const ibis::qRange &cmp,
841  const ibis::bitvector &mask,
842  ibis::bitvector &hits);
843 
844  template <typename T> static long
845  doCompare(const array_t<T> &array,
846  const ibis::qIntHod &cmp,
847  const ibis::bitvector &mask,
848  ibis::bitvector &hits);
849  template <typename T> static long
850  doCompare(const char *file,
851  const ibis::qIntHod &cmp,
852  const ibis::bitvector &mask,
853  ibis::bitvector &hits);
854  template <typename T> static long
855  negativeCompare(const array_t<T> &array,
856  const ibis::qIntHod &cmp,
857  const ibis::bitvector &mask,
858  ibis::bitvector &hits);
859  template <typename T> static long
860  negativeCompare(const char *file,
861  const ibis::qIntHod &cmp,
862  const ibis::bitvector &mask,
863  ibis::bitvector &hits);
864 
865  template <typename T> static long
866  doCompare(const array_t<T> &array,
867  const ibis::qUIntHod &cmp,
868  const ibis::bitvector &mask,
869  ibis::bitvector &hits);
870  template <typename T> static long
871  doCompare(const char *file,
872  const ibis::qUIntHod &cmp,
873  const ibis::bitvector &mask,
874  ibis::bitvector &hits);
875  template <typename T> static long
876  negativeCompare(const array_t<T> &array,
877  const ibis::qUIntHod &cmp,
878  const ibis::bitvector &mask,
879  ibis::bitvector &hits);
880  template <typename T> static long
881  negativeCompare(const char *file,
882  const ibis::qUIntHod &cmp,
883  const ibis::bitvector &mask,
884  ibis::bitvector &hits);
885 
886  template <typename T, typename F> static long
887  doComp(const array_t<T> &vals, F cmp,
888  const ibis::bitvector &mask,
889  array_t<T> &res, ibis::bitvector &hits);
890 
891  template <typename T, typename F> static long
892  doComp(const array_t<T> &vals, F cmp,
893  const ibis::bitvector &mask,
894  array_t<T> &res);
895 
896  template <typename T, typename F> static long
897  doComp(const array_t<T> &vals, F cmp,
898  const ibis::bitvector &mask,
899  ibis::bitvector &hits);
900 
901  template <typename T, typename F1, typename F2> static long
902  doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2,
903  const ibis::bitvector &mask,
904  array_t<T> &res, ibis::bitvector &hits);
905 
906  template <typename T, typename F1, typename F2> static long
907  doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2,
908  const ibis::bitvector &mask,
909  array_t<T> &res);
910 
911  template <typename T, typename F1, typename F2> static long
912  doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2,
913  const ibis::bitvector &mask,
914  ibis::bitvector &hits);
915 
916  template <typename T, typename F> static long
917  doComp0(const array_t<T> &vals, F cmp,
918  const ibis::bitvector &mask,
919  ibis::bitvector &hits);
920 
921  template <typename T, typename F1, typename F2> static long
922  doComp0(const array_t<T> &vals, F1 cmp1, F2 cmp2,
923  const ibis::bitvector &mask,
924  ibis::bitvector &hits);
925 
926  template <typename T>
927  long doCount(const ibis::qRange &cmp) const;
928 
931  template <typename T>
932  long doCount(const array_t<T> &vals, const ibis::qIntHod &cmp,
933  const ibis::bitvector &mask) const;
934 
937  template <typename T>
938  long doCount(const array_t<T> &vals, const ibis::qUIntHod &cmp,
939  const ibis::bitvector &mask) const;
940 
943  template <typename T>
944  long doCount(const array_t<T> &vals, const ibis::qRange &cmp,
945  const ibis::bitvector &mask) const;
946 
948  template <typename T, typename F>
949  long doCount(const array_t<T> &vals,
950  const ibis::bitvector &mask, F cmp) const;
951 
953  template <typename T, typename F1, typename F2>
954  long doCount(const array_t<T> &vals,
955  const ibis::bitvector &mask, F1 cmp1, F2 cmp2) const;
956 
959  long packCumulativeDistribution(const std::vector<double> &bounds,
960  const std::vector<uint32_t> &counts,
961  uint32_t nbc,
962  double *bptr, uint32_t *cptr) const;
964  long packDistribution(const std::vector<double> &bounds,
965  const std::vector<uint32_t> &counts,
966  uint32_t nbc, double *bptr, uint32_t *cptr) const;
967 
969  template <typename T1, typename T2>
970  long count2DBins(array_t<T1> &vals1,
971  const double &begin1, const double &end1,
972  const double &stride1,
973  array_t<T2> &vals2,
974  const double &begin2, const double &end2,
975  const double &stride2,
976  std::vector<uint32_t> &counts) const;
978  template <typename T1, typename T2, typename T3>
979  long count3DBins(const array_t<T1> &vals1,
980  const double &begin1, const double &end1,
981  const double &stride1,
982  const array_t<T2> &vals2,
983  const double &begin2, const double &end2,
984  const double &stride2,
985  const array_t<T3> &vals3,
986  const double &begin3, const double &end3,
987  const double &stride3,
988  std::vector<uint32_t> &counts) const;
990  template <typename T1, typename T2>
991  long count2DWeights(array_t<T1> &vals1,
992  const double &begin1, const double &end1,
993  const double &stride1,
994  array_t<T2> &vals2,
995  const double &begin2, const double &end2,
996  const double &stride2,
997  array_t<double> &wts,
998  std::vector<double> &weights) const;
1000  template <typename T1, typename T2, typename T3>
1001  long count3DWeights(const array_t<T1> &vals1,
1002  const double &begin1, const double &end1,
1003  const double &stride1,
1004  const array_t<T2> &vals2,
1005  const double &begin2, const double &end2,
1006  const double &stride2,
1007  const array_t<T3> &vals3,
1008  const double &begin3, const double &end3,
1009  const double &stride3,
1010  const array_t<double> &wts,
1011  std::vector<double> &weights) const;
1012 
1014  template <typename T1>
1015  long fill1DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
1016  const double &begin1, const double &end1,
1017  const double &stride1,
1018  std::vector<ibis::bitvector> &bins) const;
1020  template <typename T1>
1021  long fill1DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
1022  const double &begin1, const double &end1,
1023  const double &stride1,
1024  std::vector<ibis::bitvector*> &bins) const;
1026  template <typename T1>
1027  long fill1DBinsWeighted(const ibis::bitvector &mask,
1028  const array_t<T1> &vals1,
1029  const double &begin1, const double &end1,
1030  const double &stride1,
1031  const array_t<double> &wts,
1032  std::vector<double> &weights,
1033  std::vector<ibis::bitvector*> &bins) const;
1035  template <typename T1, typename T2>
1036  long fill2DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
1037  const double &begin1, const double &end1,
1038  const double &stride1,
1039  const array_t<T2> &vals2,
1040  const double &begin2, const double &end2,
1041  const double &stride2,
1042  std::vector<ibis::bitvector> &bins) const;
1043  template <typename T1>
1044  long fill2DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
1045  const double &begin1, const double &end1,
1046  const double &stride1,
1047  const ibis::column &col2,
1048  const double &begin2, const double &end2,
1049  const double &stride2,
1050  std::vector<ibis::bitvector> &bins) const;
1052  template <typename T1, typename T2>
1053  long fill2DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
1054  const double &begin1, const double &end1,
1055  const double &stride1,
1056  const array_t<T2> &vals2,
1057  const double &begin2, const double &end2,
1058  const double &stride2,
1059  std::vector<ibis::bitvector*> &bins) const;
1060  template <typename T1>
1061  long fill2DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
1062  const double &begin1, const double &end1,
1063  const double &stride1,
1064  const ibis::column &col2,
1065  const double &begin2, const double &end2,
1066  const double &stride2,
1067  std::vector<ibis::bitvector*> &bins) const;
1069  template <typename T1, typename T2>
1070  long fill2DBinsWeighted(const ibis::bitvector &mask,
1071  const array_t<T1> &vals1,
1072  const double &begin1, const double &end1,
1073  const double &stride1,
1074  const array_t<T2> &vals2,
1075  const double &begin2, const double &end2,
1076  const double &stride2,
1077  const array_t<double> &wts,
1078  std::vector<double> &weights,
1079  std::vector<ibis::bitvector*> &bins) const;
1080  template <typename T1>
1081  long fill2DBinsWeighted2(const ibis::bitvector &mask,
1082  const array_t<T1> &vals1,
1083  const double &begin1, const double &end1,
1084  const double &stride1,
1085  const ibis::column &col2,
1086  const double &begin2, const double &end2,
1087  const double &stride2,
1088  const array_t<double> &wts,
1089  std::vector<double> &weights,
1090  std::vector<ibis::bitvector*> &bins) const;
1092  template <typename T1, typename T2, typename T3>
1093  long fill3DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
1094  const double &begin1, const double &end1,
1095  const double &stride1,
1096  const array_t<T2> &vals2,
1097  const double &begin2, const double &end2,
1098  const double &stride2,
1099  const array_t<T3> &vals3,
1100  const double &begin3, const double &end3,
1101  const double &stride3,
1102  std::vector<ibis::bitvector> &bins) const;
1103  template <typename T1>
1104  long fill3DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
1105  const double &begin1, const double &end1,
1106  const double &stride1,
1107  const ibis::column &col2,
1108  const double &begin2, const double &end2,
1109  const double &stride2,
1110  const ibis::column &col3,
1111  const double &begin3, const double &end3,
1112  const double &stride3,
1113  std::vector<bitvector> &bins) const;
1114  template <typename T1, typename T2>
1115  long fill3DBins3(const ibis::bitvector &mask, const array_t<T1> &vals1,
1116  const double &begin1, const double &end1,
1117  const double &stride1,
1118  const array_t<T2> &vals2,
1119  const double &begin2, const double &end2,
1120  const double &stride2,
1121  const ibis::column &col3,
1122  const double &begin3, const double &end3,
1123  const double &stride3,
1124  std::vector<bitvector> &bins) const;
1125  template <typename T1, typename T2, typename T3>
1126  long fill3DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
1127  const double &begin1, const double &end1,
1128  const double &stride1,
1129  const array_t<T2> &vals2,
1130  const double &begin2, const double &end2,
1131  const double &stride2,
1132  const array_t<T3> &vals3,
1133  const double &begin3, const double &end3,
1134  const double &stride3,
1135  std::vector<bitvector*> &bins) const;
1136  template <typename T1>
1137  long fill3DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
1138  const double &begin1, const double &end1,
1139  const double &stride1,
1140  const ibis::column &col2,
1141  const double &begin2, const double &end2,
1142  const double &stride2,
1143  const ibis::column &col3,
1144  const double &begin3, const double &end3,
1145  const double &stride3,
1146  std::vector<ibis::bitvector*> &bins) const;
1147  template <typename T1, typename T2>
1148  long fill3DBins3(const ibis::bitvector &mask, const array_t<T1> &vals1,
1149  const double &begin1, const double &end1,
1150  const double &stride1,
1151  const array_t<T2> &vals2,
1152  const double &begin2, const double &end2,
1153  const double &stride2,
1154  const ibis::column &col3,
1155  const double &begin3, const double &end3,
1156  const double &stride3,
1157  std::vector<ibis::bitvector*> &bins) const;
1158  template <typename T1, typename T2, typename T3>
1159  long fill3DBinsWeighted(const ibis::bitvector &mask,
1160  const array_t<T1> &vals1,
1161  const double &begin1, const double &end1,
1162  const double &stride1,
1163  const array_t<T2> &vals2,
1164  const double &begin2, const double &end2,
1165  const double &stride2,
1166  const array_t<T3> &vals3,
1167  const double &begin3, const double &end3,
1168  const double &stride3,
1169  const array_t<double> &wts,
1170  std::vector<double> &weights,
1171  std::vector<bitvector*> &bins) const;
1172  template <typename T1>
1173  long fill3DBinsWeighted2(const ibis::bitvector &mask,
1174  const array_t<T1> &vals1,
1175  const double &begin1, const double &end1,
1176  const double &stride1,
1177  const ibis::column &col2,
1178  const double &begin2, const double &end2,
1179  const double &stride2,
1180  const ibis::column &col3,
1181  const double &begin3, const double &end3,
1182  const double &stride3,
1183  const array_t<double> &wts,
1184  std::vector<double> &weights,
1185  std::vector<ibis::bitvector*> &bins) const;
1186  template <typename T1, typename T2>
1187  long fill3DBinsWeighted3(const ibis::bitvector &mask,
1188  const array_t<T1> &vals1,
1189  const double &begin1, const double &end1,
1190  const double &stride1,
1191  const array_t<T2> &vals2,
1192  const double &begin2, const double &end2,
1193  const double &stride2,
1194  const ibis::column &col3,
1195  const double &begin3, const double &end3,
1196  const double &stride3,
1197  const array_t<double> &wts,
1198  std::vector<double> &weights,
1199  std::vector<ibis::bitvector*> &bins) const;
1200 
1202  long get1DBins_(const ibis::bitvector &mask, const ibis::column &col,
1203  uint32_t nbin, std::vector<double> &bounds,
1204  std::vector<ibis::bitvector> &bins, const char *mesg) const;
1205 
1207  long get1DDistribution(const ibis::column &col, uint32_t nbin,
1208  std::vector<double> &bounds,
1209  std::vector<uint32_t> &counts) const;
1211  long get2DDistributionU(const ibis::column &col1,
1212  const ibis::column &col2,
1213  uint32_t nb1, uint32_t nb2,
1214  std::vector<double> &bounds1,
1215  std::vector<double> &bounds2,
1216  std::vector<uint32_t> &counts) const;
1218  long get2DDistributionA(const ibis::column &col1,
1219  const ibis::column &col2,
1220  uint32_t nb1, uint32_t nb2,
1221  std::vector<double> &bounds1,
1222  std::vector<double> &bounds2,
1223  std::vector<uint32_t> &counts) const;
1225  long get2DDistributionI(const ibis::column &col1,
1226  const ibis::column &col2,
1227  uint32_t nb1, uint32_t nb2,
1228  std::vector<double> &bounds1,
1229  std::vector<double> &bounds2,
1230  std::vector<uint32_t> &counts) const;
1231  long old2DDistribution(const char *constraints,
1232  const char *name1, const char *name2,
1233  uint32_t nb1, uint32_t nb2,
1234  std::vector<double> &bounds1,
1235  std::vector<double> &bounds2,
1236  std::vector<uint32_t> &counts) const;
1238  int coarsenBins(const ibis::column &col, uint32_t nbin,
1239  std::vector<double> &bnds,
1240  std::vector<ibis::bitvector*> &btmp) const;
1242  long get3DDistributionA(const ibis::bitvector &mask,
1243  const ibis::column &col1,
1244  const ibis::column &col2,
1245  const ibis::column &col3,
1246  uint32_t nb1, uint32_t nb2, uint32_t nb3,
1247  std::vector<double> &bounds1,
1248  std::vector<double> &bounds2,
1249  std::vector<double> &bounds3,
1250  std::vector<uint32_t> &counts) const;
1251  template <typename E1>
1252  long get3DDistributionA1(const ibis::bitvector &mask,
1253  const array_t<E1> &vals1,
1254  const ibis::column &col2,
1255  const ibis::column &col3,
1256  uint32_t nb1, uint32_t nb2, uint32_t nb3,
1257  std::vector<double> &bounds1,
1258  std::vector<double> &bounds2,
1259  std::vector<double> &bounds3,
1260  std::vector<uint32_t> &counts) const;
1261  template <typename E1, typename E2>
1262  long get3DDistributionA2(const ibis::bitvector &mask,
1263  const array_t<E1> &vals1,
1264  const array_t<E2> &vals2,
1265  const ibis::column &col3,
1266  uint32_t nb1, uint32_t nb2, uint32_t nb3,
1267  std::vector<double> &bounds1,
1268  std::vector<double> &bounds2,
1269  std::vector<double> &bounds3,
1270  std::vector<uint32_t> &counts) const;
1271 
1272  template <typename E1, typename E2>
1273  static void mapValues(array_t<E1> &val1, array_t<E2> &val2,
1274  uint32_t nb1, uint32_t nb2,
1275  array_t<E1> &bnd1, array_t<E2> &bnd2,
1276  std::vector<uint32_t> &cnts);
1277 
1278  template <typename T>
1279  static void mapValues(const array_t<T> &vals,
1280  std::map<T, uint32_t> &hist);
1281 
1282  template <typename T>
1283  static void equalWeightBins(const array_t<T> &vals,
1284  uint32_t nbins, array_t<T> &bounds);
1285 
1286  template <typename T>
1287  static long adaptiveInts(const array_t<T> &vals, const T vmin,
1288  const T vmax, uint32_t nbins,
1289  std::vector<double> &bounds,
1290  std::vector<uint32_t> &counts);
1291 
1292  template <typename T>
1293  static long adaptiveFloats(const array_t<T> &vals, const T vmin,
1294  const T vmax, uint32_t nbins,
1295  std::vector<double> &bounds,
1296  std::vector<uint32_t> &counts);
1297 
1298  template <typename T1, typename T2>
1299  static long adaptive2DBins(const array_t<T1> &vals1,
1300  const array_t<T2> &vals2,
1301  uint32_t nb1, uint32_t nb2,
1302  std::vector<double> &bounds1,
1303  std::vector<double> &bounds2,
1304  std::vector<uint32_t> &counts);
1305 
1306  template <typename T1, typename T2, typename T3>
1307  static long adaptive3DBins(const array_t<T1> &vals1,
1308  const array_t<T2> &vals2,
1309  const array_t<T3> &vals3,
1310  uint32_t nb1, uint32_t nb2, uint32_t nb3,
1311  std::vector<double> &bounds1,
1312  std::vector<double> &bounds2,
1313  std::vector<double> &bounds3,
1314  std::vector<uint32_t> &counts);
1315 
1316  template <typename T> static long
1317  adaptiveIntsDetailed(const ibis::bitvector &mask,
1318  const array_t<T> &vals,
1319  const T vmin, const T vmax, uint32_t nbins,
1320  std::vector<double> &bounds,
1321  std::vector<ibis::bitvector> &detail);
1322 
1323  template <typename T> static long
1324  adaptiveFloatsDetailed(const ibis::bitvector &mask,
1325  const array_t<T> &vals,
1326  const T vmin, const T vmax, uint32_t nbins,
1327  std::vector<double> &bounds,
1328  std::vector<ibis::bitvector> &detail);
1329 
1330  static int reorderBitmap(ibis::bitvector &,
1331  const ibis::bitvector &,
1332  const ibis::array_t<uint32_t> &);
1333 
1334  void gatherSortKeys(ibis::table::stringArray& names);
1335  void composeQueryString(std::string &str,
1336  const ibis::column* col1, const ibis::column* col2,
1337  const double &lower1, const double &upper1,
1338  const double &lower2, const double &upper2) const;
1339  void buildQueryList(ibis::part::thrArg &lst,
1340  unsigned nc, unsigned nq) const;
1341  void checkQueryList(const ibis::part::thrArg &lst) const;
1342  uint32_t recursiveQuery(const char* pref, const column* att,
1343  double low, double high, long* nerr) const;
1344 
1345 private:
1346 
1347  /******************************************************************/
1348  // private member variables
1349  mutable pthread_mutex_t mutex;
1350  mutable pthread_rwlock_t rwlock;
1351 
1352  /******************************************************************/
1353  // private funcations
1354 
1355  void init(const char* prefix);
1356 
1357  void fillRIDs(const char* fn) const;
1358  void sortRIDs() const;
1359  uint32_t searchSortedRIDs(const ibis::rid_t &rid) const;
1360  uint32_t searchRIDs(const ibis::rid_t &rid) const;
1361  void searchSortedRIDs(const ibis::RIDSet&, ibis::bitvector&) const;
1362  void searchRIDs(const ibis::RIDSet&, ibis::bitvector&) const;
1363 
1364  // functions to perform join operations.
1365  int64_t equiJoin(const ibis::deprecatedJoin &cmp,
1366  const ibis::bitvector64 &trial,
1367  ibis::bitvector64 &result) const;
1368  int64_t deprecatedJoin(const ibis::deprecatedJoin &cmp,
1369  const ibis::bitvector64 &trial,
1370  ibis::bitvector64 &result) const;
1371  int64_t compJoin(const ibis::deprecatedJoin &cmp,
1372  const ibis::bitvector64 &trial,
1373  ibis::bitvector64 &result) const;
1374 
1375  int64_t loopJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
1376  const ibis::bitvector &mask,
1377  ibis::bitvector64 &pairs) const;
1378  int64_t loopJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
1379  const ibis::bitvector &mask) const;
1380  int64_t loopJoin(const ibis::deprecatedJoin &cmp,
1381  const ibis::bitvector &mask,
1382  ibis::bitvector64 &pairs) const;
1383  int64_t loopJoin(const ibis::deprecatedJoin &cmp,
1384  const ibis::bitvector &mask) const;
1385  int64_t equiJoinLoop1(const ibis::deprecatedJoin &cmp,
1386  const ibis::bitvector &mask,
1387  ibis::bitvector64 &pairs) const;
1388  int64_t equiJoinLoop1(const ibis::deprecatedJoin &cmp,
1389  const ibis::bitvector &mask) const;
1390  int64_t equiJoinLoop2(const ibis::deprecatedJoin &cmp,
1391  const ibis::bitvector &mask,
1392  ibis::bitvector64 &pairs) const;
1393  int64_t equiJoinLoop2(const ibis::deprecatedJoin &cmp,
1394  const ibis::bitvector &mask) const;
1395  template <class type1, class type2>
1396  void deprecatedJoinLoop(const array_t<type1> &arr1,
1397  const ibis::bitvector &msk1,
1398  const array_t<type2> &arr2,
1399  const ibis::bitvector &msk2,
1400  const double delta,
1401  ibis::bitvector64 &pairs) const;
1402  template <class type1, class type2>
1403  int64_t deprecatedJoinLoop(const array_t<type1> &arr1,
1404  const ibis::bitvector &msk1,
1405  const array_t<type2> &arr2,
1406  const ibis::bitvector &msk2,
1407  const double delta) const;
1408  int64_t deprecatedJoinLoop(const ibis::deprecatedJoin &cmp,
1409  const ibis::bitvector &mask,
1410  ibis::bitvector64 &pairs) const;
1411  int64_t deprecatedJoinLoop(const ibis::deprecatedJoin &cmp,
1412  const ibis::bitvector &mask) const;
1413  int64_t compJoinLoop(const ibis::deprecatedJoin &cmp,
1414  const ibis::bitvector &mask,
1415  ibis::bitvector64 &pairs) const;
1416  int64_t compJoinLoop(const ibis::deprecatedJoin &cmp,
1417  const ibis::bitvector &mask) const;
1418 
1419  part(const part&);
1420  const part &operator=(const part&);
1421 }; // class ibis::part
1422 
1423 namespace ibis {
1424  // Extends the name space ibis::util to contain three more functions to
1425  // deal with the reconstruction of partitions.
1426  namespace util {
1428  unsigned int FASTBIT_CXX_DLLSPEC
1429  gatherParts(ibis::partList &parts,
1430  const char *adir, const char *bdir, bool ro=false);
1432  unsigned int FASTBIT_CXX_DLLSPEC
1433  gatherParts(ibis::partList &parts, const char *adir, bool ro=false);
1436  unsigned int FASTBIT_CXX_DLLSPEC
1437  gatherParts(ibis::partList &parts, const ibis::resource &res,
1438  bool ro=false);
1439  } // namespace util
1440 } // namespace ibis
1441 
1445 struct FASTBIT_CXX_DLLSPEC ibis::part::info {
1446  const char* name;
1447  const char* description;
1448  const char* metaTags;
1449  const uint64_t nrows;
1450  std::vector<ibis::column::info*> cols;
1452 
1453  info(const char* na, const char* de, const uint64_t &nr,
1454  const ibis::part::columnList &co);
1455  info(const ibis::part &tbl);
1456  ~info();
1457 
1458 private:
1459  info(); // private default constructor, not implemented!
1460  info& operator=(const info&);
1461 }; // ibis::part::info
1462 
1464 class ibis::part::cleaner : public ibis::fileManager::cleaner {
1465 public:
1466  virtual void operator()() const;
1467  cleaner(const part* tbl) : thePart(tbl) {}
1468  virtual ~cleaner() {}
1469 
1470 private:
1471  const part* thePart;
1472 }; // ibis::part::cleaner
1473 
1478 class ibis::part::readLock {
1479 public:
1480  readLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
1481  int ierr = tbl->gainReadAccess();
1482  if (0 != ierr) {
1483  LOGGER(ibis::gVerbose > 0)
1484  << "Warning -- part[" << thePart->name()
1485  << "]::gainReadAccess -- pthread_rwlock_rdlock for " << mesg
1486  << " returned " << ierr << " (" << strerror(ierr) << ')';
1487  }
1488  else {
1489  LOGGER(ibis::gVerbose > 9)
1490  << "part[" << thePart->name()
1491  << "]::gainReadAccess for " << mesg;
1492  }
1493  }
1494  ~readLock() {
1495  int ierr = thePart->releaseAccess();
1496  if (0 != ierr) {
1497  LOGGER(ibis::gVerbose > 0)
1498  << "Warning -- part[" << thePart->name()
1499  << "]::releaseAccess -- pthread_rwlock_unlock for " << mesg
1500  << " returned " << ierr << " (" << strerror(ierr) << ')';
1501  }
1502  else {
1503  LOGGER(ibis::gVerbose > 9)
1504  << "part[" << thePart->name()
1505  << "]::releaseAccess for " << mesg;
1506  }
1507  }
1508 
1509 private:
1510  const part* thePart;
1511  const char* mesg;
1512 
1513  readLock() {}; // no default constructor
1514  readLock(const readLock&) {}; // can not copy
1515  const readLock &operator=(const readLock&);
1516 }; // ibis::part::readLock
1517 
1519 class ibis::part::writeLock {
1520 public:
1521  writeLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
1522  int ierr = tbl->gainWriteAccess();
1523  if (0 != ierr) {
1524  LOGGER(ibis::gVerbose >= 0)
1525  << "Warning -- part[" << thePart->name()
1526  << "]::gainWriteAccess -- pthread_rwlock_wrlock for " << mesg
1527  << " returned " << ierr << " (" << strerror(ierr) << ')';
1528  }
1529  else {
1530  LOGGER(ibis::gVerbose > 9)
1531  << "part[" << thePart->name()
1532  << "]::gainWriteAccess for " << mesg;
1533  }
1534  }
1535  ~writeLock() {
1536  int ierr = thePart->releaseAccess();
1537  if (0 != ierr) {
1538  LOGGER(ibis::gVerbose > 0)
1539  << "Warning -- part[" << thePart->name()
1540  << "]::releaseAccess -- pthread_rwlock_unlock for " << mesg
1541  << " returned " << ierr << " (" << strerror(ierr) << ')';
1542  }
1543  else {
1544  LOGGER(ibis::gVerbose > 9)
1545  << "part[" << thePart->name()
1546  << "]::releaseAccess for " << mesg;
1547  }
1548  }
1549 
1550 private:
1551  const part* thePart;
1552  const char* mesg;
1553 
1554  writeLock() {}; // no default constructor
1555  writeLock(const writeLock&) {}; // can not copy
1556  const writeLock &operator=(const writeLock&);
1557 }; // ibis::part::writeLock
1558 
1562 class ibis::part::softWriteLock {
1563 public:
1565  softWriteLock(const part* tbl, const char* m);
1567  ~softWriteLock();
1570  bool isLocked() const {return (lckd==0);}
1571 
1572 private:
1573  const part* thePart;
1574  const char* mesg;
1575  const int lckd;
1576 
1577  softWriteLock() : thePart(0), mesg(0), lckd(0) {};
1578  softWriteLock(const softWriteLock &rhs)
1579  : thePart(rhs.thePart), mesg(rhs.mesg), lckd(0) {};
1580  const softWriteLock &operator=(const softWriteLock&);
1581 }; // ibis::part::softWriteLock
1582 
1585 class ibis::part::mutexLock {
1586 public:
1587  mutexLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
1588  if (ibis::gVerbose > 9)
1589  tbl->logMessage("gainExclusiveAccess",
1590  "pthread_mutex_lock for %s", m);
1591  int ierr = pthread_mutex_lock(&(tbl->mutex));
1592  if (0 != ierr)
1593  tbl->logWarning("gainExclusiveAccess", "pthread_mutex_lock for %s "
1594  "returned %d (%s)", m, ierr, strerror(ierr));
1595  }
1596  ~mutexLock() {
1597  if (ibis::gVerbose > 9)
1598  thePart->logMessage("releaseExclusiveAccess",
1599  "pthread_mutex_unlock for %s", mesg);
1600  int ierr = pthread_mutex_unlock(&(thePart->mutex));
1601  if (0 != ierr)
1602  thePart->logWarning("releaseExclusiveAccess",
1603  "pthread_mutex_unlock for %s returned %d (%s)",
1604  mesg, ierr, strerror(ierr));
1605  }
1606 
1607 private:
1608  const part* thePart;
1609  const char* mesg;
1610 
1611  mutexLock() {}; // no default constructor
1612  mutexLock(const mutexLock&) {}; // can not copy
1613  const mutexLock &operator=(const mutexLock&);
1614 }; // ibis::part::mutexLock
1615 
1619 class ibis::part::barrel : public ibis::math::barrel {
1620 public:
1621  barrel(const ibis::part *t=0) : _tbl(t), position(0) {};
1622  virtual ~barrel() {close();}
1623 
1624  virtual long open(const ibis::part *t=0);
1625  virtual long close();
1626  virtual long read();
1627  virtual long seek(uint32_t pos);
1630  uint32_t tell() const {return position;}
1631 
1632  void getNullMask(ibis::bitvector &mask) const;
1633  const ibis::column* getColumn(uint32_t i) const {return cols[i];}
1634 
1635 protected:
1636  const ibis::part *_tbl;
1637  uint32_t position;
1638  std::vector<const ibis::column*> cols;
1639  std::vector<ibis::fileManager::storage*> stores;
1640  std::vector<int> fdes;
1641 }; // ibis::part::barrel
1642 
1645 class ibis::part::vault : public ibis::part::barrel {
1646 public:
1647  vault(const ibis::roster &r);
1648  virtual ~vault() {close();}
1649 
1650  virtual long open(const ibis::part *t=0);
1651  virtual long read();
1652  virtual long seek(uint32_t pos);
1653  long seek(double val);
1657  uint32_t tellReal() const;
1658 
1659 private:
1660  const ibis::roster &_roster;
1661 
1662  template <class T>
1663  uint32_t seekValue(int fd, const T &val) const;
1664  template <class T>
1665  uint32_t seekValue(const array_t<T>&arr, const T &val) const;
1666 
1667  vault();
1668  vault& operator=(const vault&);
1669 }; // ibis::part::vault
1670 /*
1671 barrel
1672 
1673 Vault \Vault\ (v[add]lt; see Note, below), n. [OE. voute, OF. voute, volte,
1674 F. vo[^u]te, LL. volta, for voluta, volutio, fr. L. volvere, volutum, to
1675 roll, to turn about. See Voluble, and cf. Vault a leap, Volt a turn,
1676 Volute.] 1. (Arch.) An arched structure of masonry, forming a ceiling or
1677 canopy.
1678 
1679 The long-drawn aisle and fretted vault. --Gray.
1680 
1681 2. An arched apartment; especially, a subterranean room, use for storing
1682  articles, for a prison, for interment, or the like; a cell; a
1683  cellar. ``Charnel vaults.'' --Milton.
1684 
1685 The silent vaults of death. --Sandys.
1686 
1687 To banish rats that haunt our vault. --Swift.
1688 
1689 3. The canopy of heaven; the sky.
1690 
1691 That heaven's vault should crack. --Shak.
1692 
1693 4. [F. volte, It. volta, originally, a turn, and the same word as volta an
1694  arch. See the Etymology above.] A leap or bound. Specifically: (a)
1695  (Man.) The bound or leap of a horse; a curvet. (b) A leap by aid of the
1696  hands, or of a pole, springboard, or the like.
1697 
1698 Note: The l in this word was formerly often suppressed in pronunciation.
1699 
1700 Barrel, Cradle, Cylindrical, or Wagon, vault (Arch.), a kind of vault
1701 having two parallel abutments, and the same section or profile at all
1702 points. It may be rampant, as over a staircase (see Rampant vault, under
1703 Rampant), or curved in plan, as around the apse of a church.
1704 
1705 Coved vault. (Arch.) See under 1st Cove, v. t.
1706 
1707 Groined vault (Arch.), a vault having groins, that is, one in which
1708 different cylindrical surfaces intersect one another, as distinguished from
1709 a barrel, or wagon, vault.
1710 
1711 Rampant vault. (Arch.) See under Rampant.
1712 
1713 Ribbed vault (Arch.), a vault differing from others in having solid ribs
1714 which bear the weight of the vaulted surface. True Gothic vaults are of
1715 this character.
1716 
1717 Vault light, a partly glazed plate inserted in a pavement or ceiling to
1718 admit light to a vault below.
1719 
1720 Source: Webster's Revised Unabridged Dictionary, (c) 1996, 1998 MICRA, Inc.
1721  */
1722 
1723 namespace ibis {
1724  // Explicit template specialization for member function
1725  // ibis::part::equalWeightBins
1726  template <> void
1727  part::equalWeightBins(const array_t<float> &vals,
1728  uint32_t nbins, array_t<float> &bounds);
1729  template <> void
1730  part::equalWeightBins(const array_t<double> &vals,
1731  uint32_t nbins, array_t<double> &bounds);
1732 
1733  template <> long
1734  part::doScan(const array_t<float> &,
1735  const ibis::qContinuousRange &,
1736  const ibis::bitvector &, ibis::bitvector &);
1737  template <> long
1738  part::doScan(const array_t<double> &,
1739  const ibis::qContinuousRange &,
1740  const ibis::bitvector &, ibis::bitvector &);
1741 
1742  template <> long
1743  part::doScan(const array_t<float> &,
1744  const ibis::qContinuousRange &,
1745  const ibis::bitvector &, ibis::array_t<float> &);
1746  template <> long
1747  part::doScan(const array_t<double> &,
1748  const ibis::qContinuousRange &,
1749  const ibis::bitvector &, ibis::array_t<double> &);
1750 
1751  template <> long
1752  part::doScan(const array_t<float> &,
1753  const ibis::qContinuousRange &,
1754  const ibis::bitvector &, ibis::array_t<float> &,
1755  ibis::bitvector &);
1756  template <> long
1757  part::doScan(const array_t<double> &,
1758  const ibis::qContinuousRange &,
1759  const ibis::bitvector &, ibis::array_t<double> &,
1760  ibis::bitvector &);
1761 
1762  template <> long
1763  part::doCount<float>(const ibis::qRange&) const;
1764  template <> long
1765  part::doCount<double>(const ibis::qRange&) const;
1766 }
1767 
1769 inline ibis::part::info* ibis::part::getInfo() const {
1770  return new info(*this);
1771 }
1772 
1777 inline ibis::column* ibis::part::getColumn(uint32_t ind) const {
1778  if (ind < columns.size()) {
1779  if (colorder.size() == columns.size()) {
1780  return const_cast<ibis::column*>(colorder[ind]);
1781  }
1782  else {
1783  ibis::part::columnList::const_iterator it = columns.begin();
1784  for (; ind > 0; -- ind) ++ it;
1785  return (*it).second;
1786  }
1787  }
1788  else {
1789  return 0;
1790  }
1791 } // ibis::part::getColumn
1792 
1793 inline const char* ibis::part::getMetaTag(const char* name) const {
1794  ibis::resource::vList::const_iterator it = metaList.find(name);
1795  if (it != metaList.end())
1796  return (*it).second;
1797  else
1798  return static_cast<const char*>(0);
1799 } // ibis::part::getMetaTag
1800 
1801 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &cmp,
1802  const ibis::bitvector &mask,
1803  ibis::bitvector64 &pairs) const {
1804  return loopJoin(cmp, mask, pairs);
1805 } // ibis::part::evaluateJoin
1806 
1807 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &,
1808  const ibis::bitvector &,
1809  const char *) const {
1810  logWarning("evaluate", "not implemented yet");
1811  return -1;
1812 } // ibis::part::evaluateJoin
1813 
1814 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &cmp,
1815  const ibis::bitvector &mask) const {
1816  return loopJoin(cmp, mask);
1817 } // ibis::part::evaluateJoin
1818 
1819 inline int64_t ibis::part::evaluateJoin
1820 (const std::vector<const ibis::deprecatedJoin*> &cmp,
1821  const ibis::bitvector &mask, ibis::bitvector64 &pairs) const {
1822  return loopJoin(cmp, mask, pairs);
1823 } // ibis::part::evaluateJoin
1824 
1828 inline int ibis::part::releaseAccess() const {
1829  return pthread_rwlock_unlock(&rwlock);
1830 } // ibis::part::releaseAccess
1831 
1835 inline int ibis::part::gainReadAccess() const {
1836  return pthread_rwlock_rdlock(&rwlock);
1837 } // ibis::part::gainReadAccess
1838 
1842 inline int ibis::part::gainWriteAccess() const {
1843  return pthread_rwlock_wrlock(&rwlock);
1844 } // ibis::part::gainWriteAccess
1845 
1849 inline int ibis::part::tryWriteAccess() const {
1850  return pthread_rwlock_trywrlock(&rwlock);
1851 } // ibis::part::tryWriteAccess
1852 #endif // IBIS_PART_H
ibis::qKeyword
The class qKeyword encapsulates a search for a single keyword in a text field.
Definition: qExpr.h:621
ibis::part::buildSorted
void buildSorted(const char *colname) const
Build a sorted version of the specified column.
Definition: part.cpp:6474
ibis::part::columnTypes
ibis::table::typeArray columnTypes() const
Return column types in a list.
Definition: part.cpp:2094
ibis::part::selectInts
array_t< int32_t > * selectInts(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as 32-bit integers.
Definition: part.cpp:2929
ibis::part::indexSpec
const char * indexSpec() const
Return the current index specification.
Definition: part.h:68
ibis::qRange
A class to represent simple range conditions.
Definition: qExpr.h:207
ibis::part::negativeScan
virtual long negativeScan(const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::bitvector &hits) const
Compute the records (marked 1 in the mask) that does not satisfy the range condition.
Definition: part.cpp:5592
ibis::part::description
const char * description() const
Return a text description of the partition.
Definition: part.h:66
ibis::part::selectDoubles
array_t< double > * selectDoubles(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as 64-bit floating-point values.
Definition: part.cpp:3159
ibis::part::softWriteLock::isLocked
bool isLocked() const
Have we acquired the desired lock? Returns true if yes, otherwise false.
Definition: part.h:1570
ibis::qAnyAny
A user specifies this type of query expression with the following syntax,.
Definition: qExpr.h:1285
ibis::part::backupDir
char * backupDir
!< The active data directory.
Definition: part.h:733
ibis::part::accessHint
ibis::fileManager::ACCESS_PREFERENCE accessHint(const ibis::bitvector &mask, unsigned elemsize=4) const
Evaluate the strategy for accessing a data file.
Definition: part.cpp:2681
ibis::part::getMetaTag
const char * getMetaTag(const char *) const
Return the value of the meta tag with the specified name.
Definition: part.h:1793
ibis::partAssoc
std::map< const char *, part *, lessi > partAssoc
An associative array for data partitions.
Definition: util.h:438
ibis::part::part
part(const char *name=0, bool ro=false)
Initialize a data partition object.
Definition: part.cpp:274
ibis::part::metaList
ibis::resource::vList metaList
!< Free form description of the partition.
Definition: part.h:728
ibis::part::getMeshDimensions
const std::vector< std::string > & getMeshDimensions() const
Return the name of the dimensions corresponding to the vector returned from getMeshShape.
Definition: part.h:578
ibis::part::readMetaData
int readMetaData(uint32_t &nrows, columnList &plist, const char *dir)
!< Don't change the data.
Definition: part.cpp:1312
ibis::part::addColumn
long addColumn(const char *aexpr, const char *cname, ibis::TYPE_T ctype=ibis::DOUBLE)
Add a column computed with the given arithmetic expression.
Definition: parti.cpp:1634
ibis::part::doBackup
void doBackup()
The routine to perform the actual copying for making a backup copy.
Definition: part.cpp:19022
ibis::part::makeBackupCopy
void makeBackupCopy()
Spawn another thread to copy the content of activeDir to backupDir.
Definition: part.cpp:19090
ibis::part::reactivate
long reactivate(const std::vector< uint32_t > &rows)
Make sure the specified rows are active.
Definition: parti.cpp:1449
ibis::part::name
const char * name() const
Return the name of the partition.
Definition: part.h:64
ibis::part::gainReadAccess
int gainReadAccess() const
Attempt to gain a read access to this part object.
Definition: part.h:1835
ibis::part::genName
static void genName(const std::vector< const char * > &mtags, std::string &name)
Generate name for a partition based on the meta tags.
Definition: part.cpp:638
ibis::part::getNullMask
void getNullMask(ibis::bitvector &m) const
Copy the mask of active rows.
Definition: part.h:618
ibis::part::count3DBins
long count3DBins(const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< T3 > &vals3, const double &begin3, const double &end3, const double &stride3, std::vector< uint32_t > &counts) const
Count the number of values in 3D bins.
Definition: parth3d.cpp:22
ibis::part::currentDataDir
const char * currentDataDir() const
Return the name of the active data directory.
Definition: part.h:82
ibis::part::state
TABLE_STATE state
!< Time of last switch operation.
Definition: part.h:735
ibis::math::barrel
A barrel to hold a list of variables.
Definition: qExpr.h:760
ibis::resource
A container for name-value pairs.
Definition: resource.h:38
ibis::part::get3DDistribution
long get3DDistribution(const char *constraints, const char *cname1, double begin1, double end1, double stride1, const char *cname2, double begin2, double end2, double stride2, const char *cname3, double begin3, double end3, double stride3, std::vector< uint32_t > &counts) const
Compute conditional 3D histogram with regularly spaced bins.
Definition: parth3d.cpp:91
ibis::part::checkQueryList
void checkQueryList(const ibis::part::thrArg &lst) const
Sum up the hits from sub-divisions to verify the hits computing from the whole range.
Definition: part.cpp:8242
ibis::part::numbersToBitvector
void numbersToBitvector(const std::vector< uint32_t > &, ibis::bitvector &) const
Turn a list of numbers into a bitvector.
Definition: part.cpp:4240
ibis::part::nColumns
uint32_t nColumns() const
Return the number of attributes in the partition.
Definition: part.h:84
ibis::part::info::description
const char * description
!< Partition name.
Definition: part.h:1447
ibis::qContinuousRange
Simple range condition.
Definition: qExpr.h:252
ibis::part::updateData
int updateData()
Check the time stamp on the metadata files to decide if the in-memory metadata information requires u...
Definition: parti.cpp:1750
ibis::part::getActualMin
double getActualMin(const char *name) const
The actual minimum value in the named column.
Definition: part.cpp:19149
ibis::qAllWords
The class qAllWords encapsulates a search for many keywords.
Definition: qExpr.h:659
ibis::part::thrArg
A struct to pack the arguments to function startTests.
Definition: part.h:642
ibis::part::stringToBitvector
void stringToBitvector(const char *, ibis::bitvector &) const
Convert a set of range conditions to an ibis::bitvector.
Definition: part.cpp:4258
ibis::part::barrel::seek
virtual long seek(uint32_t pos)
!< Read one value for each variable.
Definition: part.cpp:20063
ibis::table::namesTypes
std::map< const char *, ibis::TYPE_T, ibis::lessi > namesTypes
An associative array of names and types.
Definition: table.h:123
ibis::part::fill2DBinsWeighted
long fill2DBinsWeighted(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< double > &wts, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
Fill the bitvectors representing the 2D bins.
Definition: parth2d.cpp:2225
ibis::part::get2DDistributionI
long get2DDistributionI(const ibis::column &col1, const ibis::column &col2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
Compute 2D histogram from indexes.
Definition: parth2d.cpp:4180
ibis::part::deactivate
long deactivate(const std::vector< uint32_t > &rows)
Mark the specified rows as inactive.
Definition: parti.cpp:1410
ibis::part::buildIndexes
virtual int buildIndexes(const char *iopt, int nthr)
Make sure indexes for all columns are available.
Definition: part.cpp:6500
ibis::part::selectShorts
array_t< int16_t > * selectShorts(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as 16-bit integers.
Definition: part.cpp:2837
ibis::part::info
A simple class to describe an ibis::part object.
Definition: part.h:1445
utilidor.h
FastBit sorting functions and other utilities.
ibis::part::fill3DBinsWeighted3
long fill3DBinsWeighted3(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const ibis::column &col3, const double &begin3, const double &end3, const double &stride3, const array_t< double > &wts, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
Resolve the 3rd column involved in the 3D bins.
Definition: parth3dw.cpp:175
ibis::part::readonly
bool readonly
!< The cleaner for the file manager.
Definition: part.h:744
ibis::qIntHod
This query expression has similar meaning as ibis::qDiscreteRange, however, it stores the values as s...
Definition: qExpr.h:392
ibis::qAnyString
The column contains one of the values in a list.
Definition: qExpr.h:560
ibis::util::gatherParts
unsigned int gatherParts(ibis::partList &parts, const char *adir, const char *bdir, bool ro=false)
Look for data partitions in the given pair of directories.
Definition: part.cpp:20692
ibis::part::commit
long commit(const char *dir)
Commit the append operation involving data from dir.
Definition: parti.cpp:1082
ibis::part::skipPrefix
static const char * skipPrefix(const char *)
Skip pass all the dots in the given string.
Definition: part.cpp:6950
ibis::part::switchTime
time_t switchTime
!< The backup data directory.
Definition: part.h:734
ibis::part::estimateRange
virtual long estimateRange(const ibis::qContinuousRange &cmp) const
Return an upper bound on the number of hits.
Definition: part.cpp:3664
ibis::part::readMeshShape
void readMeshShape(const char *const dir)
Read shape of the mesh from the metadata file.
Definition: part.cpp:1229
ibis::part::info::metaTags
const char * metaTags
!< A free-form description of the partition.
Definition: part.h:1448
ibis::roster
A roster is a list of values in ascending order plus their original positions.
Definition: iroster.h:19
ibis::part::rollback
long rollback()
Rollback the append operation.
Definition: parti.cpp:984
ibis
The current implementation of FastBit is code named IBIS; most data structures and functions are in t...
Definition: bord.h:16
ibis::part::writeOpaques
static int writeOpaques(int bdes, int sdes, ibis::bitvector::word_t nold, ibis::bitvector::word_t nnew, ibis::bitvector::word_t voffset, const std::vector< ibis::opaque > &opq, ibis::bitvector &totmask, const ibis::bitvector &newmask)
Write raw bytes to an open file.
Definition: part.cpp:19499
ibis::part::estimateCost
virtual double estimateCost(const ibis::qContinuousRange &cmp) const
Estimate the cost of evaluate the query expression.
Definition: part.cpp:3694
ibis::part::adaptive2DBins
static long adaptive2DBins(const array_t< T1 > &vals1, const array_t< T2 > &vals2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts)
Adaptive binning through regularly spaced bins.
Definition: parth2d.cpp:2889
ibis::column
The class to represent a column of a data partition.
Definition: column.h:65
ibis::part::adaptiveInts
static long adaptiveInts(const array_t< T > &vals, const T vmin, const T vmax, uint32_t nbins, std::vector< double > &bounds, std::vector< uint32_t > &counts)
The adaptive binning function for integer values.
Definition: parth.cpp:2036
ibis::part::adaptiveFloats
static long adaptiveFloats(const array_t< T > &vals, const T vmin, const T vmax, uint32_t nbins, std::vector< double > &bounds, std::vector< uint32_t > &counts)
The adaptive binning function for floats and integers in wide ranges.
Definition: parth.cpp:2134
ibis::part::activeDir
char * activeDir
!< Number of events (rows) in the partition.
Definition: part.h:732
ibis::part::freeRIDs
void freeRIDs() const
!< Read RIDs from file 'rids'.
Definition: part.cpp:2200
ibis::part::setMeshShape
void setMeshShape(const ibis::array_t< uint64_t > &)
copy the incoming as the mesh shape of the data partition.
Definition: part.cpp:1874
ibis::bitvector64
A data structure to represent a sequence of bits.
Definition: bitvector64.h:54
ibis::part::~part
virtual ~part()
Destuctor.
Definition: part.cpp:600
ibis::part::getColumnSum
double getColumnSum(const char *name) const
Sum of all value in the named column.
Definition: part.cpp:19165
ibis::part::releaseAccess
int releaseAccess() const
Attempt to release a read or write access to this part object.
Definition: part.h:1828
ibis::part::evaluateRIDSet
long evaluateRIDSet(const ibis::RIDSet &, ibis::bitvector &) const
Convert a list of RIDs into a bitvector.
Definition: part.cpp:3287
ibis::array_t::push_back
void push_back(const T &elm)
Add one element from the back.
Definition: array_t.h:204
ibis::part::vault
To read variables in certain order.
Definition: part.h:1645
ibis::part::get3DDistributionA2
long get3DDistributionA2(const ibis::bitvector &mask, const array_t< E1 > &vals1, const array_t< E2 > &vals2, const ibis::column &col3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< uint32_t > &counts) const
Read the values of the third column.
Definition: parth3d.cpp:5749
ibis::part::timestamp
time_t timestamp() const
Return the time stamp on the partition.
Definition: part.h:72
ibis::bitvector::copy
bitvector & copy(const bitvector &bv)
Make a copy. Performs a deep copy.
Definition: bitvector.h:680
ibis::part::adaptive3DBins
static long adaptive3DBins(const array_t< T1 > &vals1, const array_t< T2 > &vals2, const array_t< T3 > &vals3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< uint32_t > &counts)
Adaptive binning through regularly spaced bins.
Definition: parth3d.cpp:5002
ibis::part::selectFloats
array_t< float > * selectFloats(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as 32-bit floating-point values.
Definition: part.cpp:3113
ibis::part::selfTest
virtual long selfTest(int nth=1, const char *pref=0) const
Perform predefined set of tests and return the number of failures.
Definition: part.cpp:6963
ibis::part::buildQueryList
void buildQueryList(ibis::part::thrArg &lst, unsigned nc, unsigned nq) const
Generate a list of random query conditions.
Definition: part.cpp:8039
ibis::part::packCumulativeDistribution
long packCumulativeDistribution(const std::vector< double > &bounds, const std::vector< uint32_t > &counts, uint32_t nbc, double *bptr, uint32_t *cptr) const
Pack a cumulative distribution stored in two std::vectors into two arrays provided by the caller...
Definition: parth.cpp:4228
ibis::qString
The class qString encapsulates information for comparing string values.
Definition: qExpr.h:504
ibis::part::stringSearch
long stringSearch(const ibis::qString &cmp, ibis::bitvector &low) const
Find all records that has the exact string value.
Definition: part.cpp:3323
ibis::part::selectUBytes
array_t< unsigned char > * selectUBytes(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as 8-bit unsigned integers.
Definition: part.cpp:2791
ibis::part::getJointDistribution
long getJointDistribution(const char *constraints, const char *name1, const char *name2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
Compute the joint distribution of two variables.
Definition: parth2d.cpp:5933
ibis::part::reorderBitmap
static int reorderBitmap(ibis::bitvector &, const ibis::bitvector &, const ibis::array_t< uint32_t > &)
Produce a reordered bit vector through the inverse order array.
Definition: parti.cpp:672
ibis::part::doScan
virtual long doScan(const ibis::qRange &cmp, ibis::bitvector &hits) const
Evaluate the range condition.
Definition: part.cpp:4273
ibis::part::writeValues
long writeValues(const char *fname, const array_t< uint32_t > &ind)
Write the named data file with values in the given order.
Definition: parti.cpp:456
ibis::part::readLock
Provide a read lock on an ibis::part.
Definition: part.h:1478
ibis::part::writeLock
Provide a write lock on an ibis::part.
Definition: part.h:1519
ibis::part::evaluateJoin
int64_t evaluateJoin(const ibis::deprecatedJoin &cmp, const ibis::bitvector &mask, ibis::bitvector64 &pairs) const
Evaluate a self-join.
Definition: part.h:1801
ibis::part::getUndecidable
virtual float getUndecidable(const ibis::qContinuousRange &cmp, ibis::bitvector &iffy) const
Discover the records that can not be decided using the index.
Definition: part.cpp:5880
ibis::part::fill3DBins
long fill3DBins(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< T3 > &vals3, const double &begin3, const double &end3, const double &stride3, std::vector< ibis::bitvector > &bins) const
Fill the bitvectors representing the 3D bins.
Definition: parth3db.cpp:26
ibis::part::getColumn
column * getColumn(const char *name) const
Given a name, return the associated column.
Definition: part.cpp:6876
ibis::part::colorder
std::vector< const column * > colorder
!< Active rows are maked 1.
Definition: part.h:739
ibis::part::purgeInactive
long purgeInactive()
Purge all inactive rows from the partition.
Definition: parti.cpp:1485
ibis::part::cleaner::operator()
virtual void operator()() const
Unload the indexes to free up some resources.
Definition: part.cpp:19669
ibis::part::doCount
long doCount(const ibis::qRange &cmp) const
Count the number rows satisfying the range expression.
Definition: part.cpp:17475
ibis::part::fill2DBins
long fill2DBins(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, std::vector< ibis::bitvector > &bins) const
Fill the bitvectors representing the 2D bins.
Definition: parth2d.cpp:1038
ibis::part::patternSearch
long patternSearch(const ibis::qLike &cmp, ibis::bitvector &low) const
Look for string like the given pattern.
Definition: part.cpp:3447
ibis::part::getMeshShape
const std::vector< uint32_t > & getMeshShape() const
In many scientific applications, data are defined on meshes.
Definition: part.h:575
ibis::part::append2
long append2(const char *dir)
Perform append operation with two data directories.
Definition: parti.cpp:854
ibis::part::getCumulativeDistribution
long getCumulativeDistribution(const char *name, std::vector< double > &bounds, std::vector< uint32_t > &counts) const
Compute a cumulative distribution (a cumulative histogram).
Definition: parth.cpp:3908
ibis::part::extendMetaTags
void extendMetaTags()
!< Remove the rids list from memory.
ibis::part::recursiveQuery
uint32_t recursiveQuery(const char *pref, const column *att, double low, double high, long *nerr) const
Issues a query and then subdivided the range into three to check the total hits of the three sub quer...
Definition: part.cpp:7826
ibis::part::softWriteLock::~softWriteLock
~softWriteLock()
Destructor.
Definition: parti.cpp:1882
ibis::part::count2DWeights
long count2DWeights(array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, array_t< double > &wts, std::vector< double > &weights) const
Count the weights in 2D bins.
Definition: parth2d.cpp:516
ibis::part::gainWriteAccess
int gainWriteAccess() const
Attempt to gain a write access to this part object.
Definition: part.h:1842
ibis::util::logMessage
void logMessage(const char *event, const char *fmt,...)
Print a message to standard output.
Definition: util.cpp:1902
ibis::part::columnList
std::map< const char *, column *, lessi > columnList
!< A function to start backing up the active dir.
Definition: part.h:673
ibis::part::count3DWeights
long count3DWeights(const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< T3 > &vals3, const double &begin3, const double &end3, const double &stride3, const array_t< double > &wts, std::vector< double > &weights) const
Count the weights in 3D bins.
Definition: parth3d.cpp:2437
ibis::part::vault::open
virtual long open(const ibis::part *t=0)
The function valut::open different from barrel::open in that it opens the .srt file for the first var...
Definition: part.cpp:20098
ibis::part::thrArg::super
std::vector< unsigned > super
!< List of query conditions.
Definition: part.h:648
ibis::part::matchNameValuePair
bool matchNameValuePair(const char *name, const char *value) const
Match a name-value pair in the meta tags.
Definition: part.cpp:1924
ibis::part::testRangeOperators
void testRangeOperators(const ibis::column *col, long *nerrors) const
Try a set of range conditions with different combinations of operators.
Definition: part.cpp:7747
ibis::part::selectULongs
array_t< uint64_t > * selectULongs(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as 64-bit unsigned integers.
Definition: part.cpp:3067
ibis::part::vault::read
virtual long read()
!< Open all data files.
Definition: part.cpp:20251
ibis::part::selectUShorts
array_t< uint16_t > * selectUShorts(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as 16-bit unsigned integers.
Definition: part.cpp:2883
ibis::array_t< const char * >
ibis::fileManager::ACCESS_PREFERENCE
ACCESS_PREFERENCE
Hint passed to the function getFile.
Definition: fileManager.h:29
ibis::init
void init(const char *rcfile=0, const char *mesgfile=0)
Initializes the memory manager of FastBit.
Definition: ibis.h:378
ibis::part::purgeIndexFiles
void purgeIndexFiles() const
Remove existing index files! The indexes will be rebuilt next time they are needed.
Definition: part.cpp:6845
ibis::part::get3DBins
long get3DBins(const char *constraints, const char *cname1, double begin1, double end1, double stride1, const char *cname2, double begin2, double end2, double stride2, const char *cname3, double begin3, double end3, double stride3, std::vector< ibis::bitvector > &bins) const
Partition values of named variables into regularly spaced 3D bins.
Definition: parth3db.cpp:644
resource.h
Defines a class to hold name-value pairs.
ibis::DOUBLE
!< Four-byte IEEE floating-point numbers, internally float.
Definition: table.h:39
ibis::part::getState
TABLE_STATE getState() const
Retrieve the current state of data partition.
Definition: part.cpp:6868
ibis::part::cleaner
A cleaner to be used by the function fileManager::unload.
Definition: part.h:1464
ibis::part::packDistribution
long packDistribution(const std::vector< double > &bounds, const std::vector< uint32_t > &counts, uint32_t nbc, double *bptr, uint32_t *cptr) const
Pack a binned distribution.
Definition: parth.cpp:4146
ibis::part
The class ibis::part represents a partition of a relational table.
Definition: part.h:27
ibis::TYPE_T
TYPE_T
Supported data types.
Definition: table.h:25
ibis::qUIntHod
This query expression has similar meaning as ibis::qDiscreteRange, however, it stores the values as u...
Definition: qExpr.h:447
ibis::part::adaptiveIntsDetailed
static long adaptiveIntsDetailed(const ibis::bitvector &mask, const array_t< T > &vals, const T vmin, const T vmax, uint32_t nbins, std::vector< double > &bounds, std::vector< ibis::bitvector > &detail)
Bins the given values so that each each bin is nearly equal weight.
Definition: parth.cpp:2207
ibis::part::print
void print(std::ostream &out) const
Output a description of every column in the data partition.
Definition: part.cpp:2129
ibis::util::counter
A simple shared counter.
Definition: util.h:869
ibis::part::gatherSortKeys
void gatherSortKeys(ibis::table::stringArray &names)
Collect a list of column names that might be used as keys for sorting the rows.
Definition: parti.cpp:55
ibis::part::softWriteLock
An non-blocking version of writeLock.
Definition: part.h:1562
ibis::part::mapValues
static void mapValues(array_t< E1 > &val1, array_t< E2 > &val2, uint32_t nb1, uint32_t nb2, array_t< E1 > &bnd1, array_t< E2 > &bnd2, std::vector< uint32_t > &cnts)
The templated function to decide the bin boundaries and count the number of values fall in each bin...
Definition: parth2d.cpp:5733
ibis::part::getStateNoLocking
TABLE_STATE getStateNoLocking() const
Return the current state of data partition.
Definition: part.h:54
ibis::part::coarsenBins
int coarsenBins(const ibis::column &col, uint32_t nbin, std::vector< double > &bnds, std::vector< ibis::bitvector * > &btmp) const
Produce a set of bitmaps corresponding to a set of coarse bins.
Definition: parth.cpp:2482
ibis::part::estimateMatchAny
virtual long estimateMatchAny(const ibis::qAnyAny &cmp, ibis::bitvector &low, ibis::bitvector &high) const
Estimate a lower bound and an upper bound on the records that are hits.
Definition: part.cpp:4169
ibis::part::get1DBins_
long get1DBins_(const ibis::bitvector &mask, const ibis::column &col, uint32_t nbin, std::vector< double > &bounds, std::vector< ibis::bitvector > &bins, const char *mesg) const
Compute 1D histogram from raw data.
Definition: parth.cpp:2607
ibis::part::append
long append(const char *dir)
Append data from dir.
Definition: parti.cpp:711
ibis::part::fill3DBinsWeighted2
long fill3DBinsWeighted2(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const ibis::column &col2, const double &begin2, const double &end2, const double &stride2, const ibis::column &col3, const double &begin3, const double &end3, const double &stride3, const array_t< double > &wts, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
Resolve the 2nd column of the 3D bins.
Definition: parth3dw.cpp:424
ibis::part::get3DDistributionA1
long get3DDistributionA1(const ibis::bitvector &mask, const array_t< E1 > &vals1, const ibis::column &col2, const ibis::column &col3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< uint32_t > &counts) const
Read the value of the second column.
Definition: parth3d.cpp:5581
ibis::part::getRowNumber
uint32_t getRowNumber(const rid_t &rid) const
Return the row number of the row with specified RID.
Definition: part.cpp:2323
ibis::part::writeMetaData
void writeMetaData(const uint32_t nrows, const columnList &plist, const char *dir) const
Write metadata file -part.txt.
Definition: part.cpp:1722
ibis::math::stringFunction1
One-argument string functions.
Definition: qExpr.h:1110
ibis::part::m_desc
std::string m_desc
!< Name of the data partition.
Definition: part.h:727
ibis::part::updateMetaData
void updateMetaData() const
Write the metadata file to record the changes to the partition.
Definition: part.cpp:1837
ibis::part::keywordSearch
long keywordSearch(const ibis::qKeyword &cmp, ibis::bitvector &low) const
Identify all rows containing the specified keyword.
Definition: part.cpp:3475
ibis::part::get3DDistributionA
long get3DDistributionA(const ibis::bitvector &mask, const ibis::column &col1, const ibis::column &col2, const ibis::column &col3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< uint32_t > &counts) const
Compute 3D histogram with adaptive bins from base data.
Definition: parth3d.cpp:5413
ibis::part::get2DDistributionA
long get2DDistributionA(const ibis::column &col1, const ibis::column &col2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
Compute 2D histogram with adaptive bins from base data.
Definition: parth2d.cpp:3159
ibis::part::computeMinMax
void computeMinMax()
Compute the min and max for each column.
Definition: part.cpp:6453
ibis::part::vault::seek
virtual long seek(uint32_t pos)
!< Read the values at the current position.
Definition: part.cpp:20406
ibis::part::doComp0
static long doComp0(const array_t< T > &vals, F cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
Evaluate the range condition.
Definition: part.cpp:16891
ibis::part::get1DDistribution
long get1DDistribution(const char *constraints, const char *cname, double begin, double end, double stride, std::vector< uint32_t > &counts) const
Histogram functions.
Definition: parth.cpp:57
ibis::util::sortRIDs
void sortRIDs(ibis::RIDSet &)
Sort RID lists.
Definition: utilidor.cpp:117
ibis::part::fill2DBins2
long fill2DBins2(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const ibis::column &col2, const double &begin2, const double &end2, const double &stride2, std::vector< ibis::bitvector > &bins) const
A template function to resolve the second variable involved in the 2D bins.
Definition: parth2d.cpp:1121
ibis::part::get2DBins
long get2DBins(const char *constraints, const char *cname1, double begin1, double end1, double stride1, const char *cname2, double begin2, double end2, double stride2, std::vector< ibis::bitvector > &bins) const
Partition values of named variables into regularly spaced 2D bins.
Definition: parth2d.cpp:1356
ibis::part::getMaskRef
const ibis::bitvector & getMaskRef() const
Return a reference to the mask of active rows.
Definition: part.h:620
ibis::part::getActualMax
double getActualMax(const char *name) const
The actual maximum value in the named column.
Definition: part.cpp:19157
ibis::part::selectUInts
array_t< uint32_t > * selectUInts(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as 32-bit unsigned integers.
Definition: part.cpp:2975
ibis::part::logError
void logError(const char *event, const char *fmt,...) const
Write out a message with indication of severe error.
Definition: part.cpp:8268
ibis::part::evaluateRange
virtual long evaluateRange(const ibis::qContinuousRange &cmp, const ibis::bitvector &mask, ibis::bitvector &res) const
Evaluate a continue range expression accurately.
Definition: part.cpp:3568
ibis::part::barrel::read
virtual long read()
!< Close all open files.
Definition: part.cpp:19910
ibis::part::selectBytes
array_t< signed char > * selectBytes(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as 8-bit integers.
Definition: part.cpp:2745
ibis::part::old2DDistribution
long old2DDistribution(const char *constraints, const char *name1, const char *name2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
The old implementation that uses binary lookup.
Definition: parth2d.cpp:5170
ibis::part::writeRaw
static int writeRaw(int bdes, int sdes, ibis::bitvector::word_t nold, ibis::bitvector::word_t nnew, ibis::bitvector::word_t voffset, const ibis::array_t< unsigned char > &bytes, const ibis::array_t< int64_t > &starts, ibis::bitvector &totmask, const ibis::bitvector &newmask)
Write raw bytes to an open file.
Definition: part.cpp:19327
ibis::part::readRIDs
void readRIDs() const
A function to retrieve RIDs stored in file.
Definition: part.cpp:2171
ibis::part::loadIndexes
void loadIndexes(const char *iopt=0, int ropt=0) const
Load indexes of all columns.
Definition: part.cpp:6743
ibis::part::calculate
long calculate(const ibis::math::term &, const ibis::bitvector &, array_t< double > &) const
Calculate the values of an arithmetic expression as doubles.
Definition: part.cpp:6120
ibis::part::tryWriteAccess
int tryWriteAccess() const
A soft attempt to gain a write access to this part object.
Definition: part.h:1849
ibis::part::selectStrings
std::vector< std::string > * selectStrings(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as strings.
Definition: part.cpp:3205
ibis::part::readMetaTags
static char * readMetaTags(const char *const dir)
A class function to read the meta tags in the tdc file.
Definition: part.cpp:1154
ibis::part::doCompare
static long doCompare(const array_t< T > &array, const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::array_t< T > &res, ibis::bitvector &hits)
The function that performs the actual comparison for range queries.
Definition: part.cpp:9319
ibis::part::countPages
static uint32_t countPages(const ibis::bitvector &mask, unsigned elemsize=4)
Estimate the number of pages to be accessed.
Definition: part.cpp:2616
ibis::part::columnNames
ibis::table::stringArray columnNames() const
Return column names in a list.
Definition: part.cpp:2060
ibis::math::term
The abstract base class for arithmetic terms.
Definition: qExpr.h:728
ibis::part::fill1DBinsWeighted
long fill1DBinsWeighted(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< double > &wts, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
Fill the bitvectors representing the 1D bins.
Definition: parth.cpp:1118
ibis::part::selectValues
long selectValues(const char *cname, const ibis::bitvector &mask, void *vals) const
Select values of a column based on the given mask.
Definition: part.cpp:3248
ibis::part::combineNames
void combineNames(ibis::table::namesTypes &metalist) const
Update the list of columns with information in this data partition.
Definition: part.cpp:2037
ibis::part::clear
int clear()
A group of functions to manipulate the data partition.
Definition: parti.cpp:1814
ibis::part::matchMetaTags
bool matchMetaTags(const std::vector< const char * > &mtags) const
Match multiple name-value pairs against the internally stored meta tags.
Definition: part.cpp:1949
ibis::part::reorderValues
long reorderValues(const char *fname, array_t< uint32_t > &starts, array_t< uint32_t > &indout, const array_t< uint32_t > &indin, bool ascending)
Write the named data file in a segmented sorted order.
Definition: parti.cpp:529
ibis::part::mutexLock
Provide a mutual exclusion lock on an ibis::part object.
Definition: part.h:1585
ibis::bitvector
A data structure to represent a sequence of bits.
Definition: bitvector.h:62
ibis::part::vault::tellReal
uint32_t tellReal() const
Tell the physical record number.
Definition: part.cpp:20557
ibis::part::barrel::~barrel
virtual ~barrel()
Destructor. Member variables clean themselves.
Definition: part.h:1622
ibis::part::info::nrows
const uint64_t nrows
!< A string of name-value pairs.
Definition: part.h:1449
ibis::part::queryTest
void queryTest(const char *pref, long *nerrors) const
Generate and run random queries for slefTest.
Definition: part.cpp:7225
ibis::part::fill3DBins3
long fill3DBins3(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const ibis::column &col3, const double &begin3, const double &end3, const double &stride3, std::vector< bitvector > &bins) const
Resolve the 3rd column involved in the 3D bins.
Definition: parth3db.cpp:159
ibis::part::fill2DBinsWeighted2
long fill2DBinsWeighted2(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const ibis::column &col2, const double &begin2, const double &end2, const double &stride2, const array_t< double > &wts, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
This version returns a vector of pointers to bitmaps.
Definition: parth2d.cpp:2329
ibis::part::rids
array_t< rid_t > * rids
!< Meta tags as name-value pairs.
Definition: part.h:729
ibis::part::myCleaner
ibis::part::cleaner * myCleaner
!< Sizes of the dimensions.
Definition: part.h:743
ibis::part::writeColumn
static int writeColumn(int fdes, ibis::bitvector::word_t nold, ibis::bitvector::word_t nnew, ibis::bitvector::word_t voffset, const array_t< T > &vals, const T &fill, ibis::bitvector &totmask, const ibis::bitvector &newmask)
Write the content of vals to an open file.
Definition: part.cpp:19181
ibis::part::doComp
static long doComp(const array_t< T > &vals, F cmp, const ibis::bitvector &mask, array_t< T > &res, ibis::bitvector &hits)
Evaluate the range condition.
Definition: part.cpp:17175
ibis::part::fill3DBins2
long fill3DBins2(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const ibis::column &col2, const double &begin2, const double &end2, const double &stride2, const ibis::column &col3, const double &begin3, const double &end3, const double &stride3, std::vector< bitvector > &bins) const
Resolve the 2nd column of the 3D bins.
Definition: parth3db.cpp:396
ibis::part::amask
ibis::bitvector amask
!< Index specification.
Definition: part.h:738
ibis::part::countHits
long countHits(const ibis::qRange &cmp) const
Count the number of hits for a single range condition.
Definition: part.cpp:17408
ibis::part::getInfo
info * getInfo() const
Return an ibis::part::info object that describes the current partition.
Definition: part.h:1769
ibis::part::append1
long append1(const char *dir)
Perform append operation using only only one data directory.
Definition: parti.cpp:768
ibis::part::fill1DBins
long fill1DBins(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, std::vector< ibis::bitvector > &bins) const
Fill the bitvectors representing the 1D bins.
Definition: parth.cpp:401
ibis::part::selectLongs
array_t< int64_t > * selectLongs(const char *name, const ibis::bitvector &mask) const
Retrieve values of the named column as 64-bit integers.
Definition: part.cpp:3021
ibis::part::getDistribution
long getDistribution(const char *name, std::vector< double > &bounds, std::vector< uint32_t > &counts) const
Obsolete histogram functions.
Definition: parth.cpp:3553
ibis::qDiscreteRange
A discrete range expression.
Definition: qExpr.h:337
ibis::part::quickTest
void quickTest(const char *pref, long *nerrors) const
Generate and run random queries for slefTest.
Definition: part.cpp:7320
ibis::part::shapeSize
std::vector< uint32_t > shapeSize
!< Names of the dimensions.
Definition: part.h:741
ibis::part::shapeName
std::vector< std::string > shapeName
!< An ordering of columns.
Definition: part.h:740
ibis::part::thrArg::hits
std::vector< unsigned > hits
!< The condition encompassing this one.
Definition: part.h:649
ibis::part::indexBuilderPool
A struct to pack arguments to the function ibis_part_build_index.
Definition: part.h:652
ibis::part::writeStrings
static int writeStrings(const char *fnm, ibis::bitvector::word_t nold, ibis::bitvector::word_t nnew, ibis::bitvector::word_t voffset, const std::vector< std::string > &vals, ibis::bitvector &totmask, const ibis::bitvector &newmask)
Write strings to an open file.
Definition: part.cpp:19246
ibis::part::adaptiveFloatsDetailed
static long adaptiveFloatsDetailed(const ibis::bitvector &mask, const array_t< T > &vals, const T vmin, const T vmax, uint32_t nbins, std::vector< double > &bounds, std::vector< ibis::bitvector > &detail)
Bins the given values so that each each bin is nearly equal weight.
Definition: parth.cpp:2356
ibis::part::rename
void rename(const ibis::partAssoc &known)
Rename the partition to avoid conflicts with an existing list of data partitions. ...
Definition: part.cpp:686
ibis::fileManager::cleaner
A function object to be used to register external cleaners.
Definition: fileManager.h:66
ibis::part::barrel::close
virtual long close()
!< Open all data files.
Definition: part.cpp:19894
ibis::part::nRows
uint32_t nRows() const
Return the number of rows.
Definition: part.h:86
ibis::part::digestMeshShape
void digestMeshShape(const char *shape)
Convert the string describing the shape into internal storage format.
Definition: part.cpp:1978
ibis::part::fill3DBinsWeighted
long fill3DBinsWeighted(const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< T3 > &vals3, const double &begin3, const double &end3, const double &stride3, const array_t< double > &wts, std::vector< double > &weights, std::vector< bitvector * > &bins) const
The three triplets, (begin1, end1, stride1), (begin2, end2, stride2), and (begin3, end3, stride3), defines (1 + floor((end1 - begin1) / stride1)) (1 + floor((end2 - begin2) / stride2)) (1 + floor((end3 - begin3) / stride3)) 3D bins.
Definition: parth3dw.cpp:27
ibis::part::appendToBackup
long appendToBackup(const char *dir)
Append data in dir to the partition in the backup directory.
Definition: parti.cpp:1154
ibis::part::nEvents
uint32_t nEvents
!< List of the columns.
Definition: part.h:731
ibis::part::explicitRIDs
bool explicitRIDs() const
Does this partition have an explicit RID column? Returns true for yes, false for no.
Definition: part.h:120
ibis::qLike
Representing the operator 'LIKE'.
Definition: qExpr.h:585
ibis::part::emptyCache
void emptyCache() const
Empty all unused resources in cache.
Definition: parti.cpp:1619
ibis::part::reorder
virtual long reorder()
Sort rows with the lowest cardinality column first.
Definition: parti.cpp:31
ibis::part::metaTags
std::string metaTags() const
Return the list of meta tags as a single string.
Definition: part.cpp:1907
ibis::part::thrArg::cnt
ibis::util::counter cnt
!< Number of errors encountered.
Definition: part.h:646
ibis::part::barrel::getNullMask
void getNullMask(ibis::bitvector &mask) const
Collect the null masks together.
Definition: part.cpp:19709
column.h
Define the class ibis::column.
ibis::part::count2DBins
long count2DBins(array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, std::vector< uint32_t > &counts) const
Count the number of values in 2D bins.
Definition: parth2d.cpp:22
_rwlock
Definition: const.h:299
ibis::part::get2DDistribution
long get2DDistribution(const char *constraints, const char *cname1, double begin1, double end1, double stride1, const char *cname2, double begin2, double end2, double stride2, std::vector< uint32_t > &counts) const
Compute conditional 2D histogram with regularly spaced bins.
Definition: parth2d.cpp:54
ibis::part::info::cols
std::vector< ibis::column::info * > cols
!< The number of rows in the partition.
Definition: part.h:1451
ibis::part::barrel
To read a list of variables at the same time.
Definition: part.h:1619
ibis::part::columns
columnList columns
!< The object IDs (row id).
Definition: part.h:730
ibis::part::get2DDistributionU
long get2DDistributionU(const ibis::column &col1, const ibis::column &col2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
Compute 2D histogram with uniform bins from base data.
Definition: parth2d.cpp:3643
ibis::math::barrel::barrel
barrel()
Constructor.
Definition: qExpr.h:763
ibis::compRange
The class compRange stores computed ranges.
Definition: qExpr.h:1167
ibis::part::get1DBins
long get1DBins(const char *constraints, const char *cname, double begin, double end, double stride, std::vector< ibis::bitvector > &bins) const
Partition values of the named variable into regularly spaced bins.
Definition: parth.cpp:478
ibis::deprecatedJoin
A join is defined by two names and a numerical expression.
Definition: qExpr.h:1240
ibis::part::barrel::open
virtual long open(const ibis::part *t=0)
!< Destructor closes the open files.
Definition: part.cpp:19749
ibis::part::setMetaTags
void setMetaTags(const ibis::resource::vList &mts)
Make a deep copy of the incoming name-value pairs.
Definition: part.cpp:1881
ibis::part::unloadIndexes
void unloadIndexes() const
Unload indexes of all columns.
Definition: part.cpp:6832

Make It A Bit Faster
Contact us
Disclaimers
FastBit source code
FastBit mailing list archive