A two-level index. More...

#include <ibin.h>

Inheritance diagram for ibis::fuge:

Public Member Functions
virtual void	adjustLength (uint32_t nrows)
	Fill the bitmaps to the specified size. More...

virtual long	append (const char dt, const char df, uint32_t nnew)
	Create index for the data in df and append the result to the index in dt.

long	append (const ibis::fuge &tail)

virtual void	estimate (const ibis::qContinuousRange &expr, ibis::bitvector &lower, ibis::bitvector &upper) const
	Provide an estimation based on the current index. More...

virtual long	evaluate (const ibis::qContinuousRange &expr, ibis::bitvector &hits) const
	To evaluate the exact hits. More...

virtual long	evaluate (const ibis::qDiscreteRange &expr, ibis::bitvector &hits) const
	To evaluate the exact hits. More...

	fuge (const ibis::column c, ibis::fileManager::storage st, size_t offset=8)
	Reconstruct from content of a file. More...

	fuge (const ibis::column , const char )

	fuge (const ibis::bin &rhs)

virtual const char *	name () const
	Returns the name of the index, similar to the function `type`, but returns a string instead. More...

virtual void	print (std::ostream &out) const
	Prints human readable information. More...

virtual int	read (const char *idxfile)
	Read the content of the named file.

virtual int	read (ibis::fileManager::storage *st)
	Read an index from the storage object.

virtual INDEX_TYPE	type () const
	Returns an index type identifier.

virtual int	write (const char *dt) const
	Write the index to the named directory or file.

Public Member Functions inherited from ibis::bin
long	append (const ibis::bin &tail)
	Append the `tail` to this index.

long	append (const array_t< uint32_t > &ind)
	Append a list of integers representing bin numbers.

	bin (const ibis::bin &rhs)
	Copy constructor. It performs a deep copy.

	bin (const ibis::column c=0, const char f=0)
	Constructor. Construct a bitmap index from current data.

	bin (const ibis::column c, ibis::fileManager::storage st, size_t offset=8)

	bin (const ibis::column c, const char f, const array_t< double > &bd)
	Constructor. Construct an index with the given bin boundaries.

	bin (const ibis::column c, const char f, const std::vector< double > &bd)
	Constructor. Construct an index with the given bin boundaries.

	bin (const ibis::column c, uint32_t nb, double keys, int64_t *offs)
	Reconstruct an object from keys and offsets.

	bin (const ibis::column c, uint32_t nb, double keys, int64_t offs, uint32_t bms)
	Reconstruct an object from keys and offsets.

	bin (const ibis::column c, uint32_t nb, double keys, int64_t offs, void bms, FastBitReadBitmaps rd)
	Reconstruct an object from keys and offsets.

virtual void	binBoundaries (std::vector< double > &) const
	The function binBoundaries and binWeights return bin boundaries and counts of each bin respectively. More...

virtual void	binWeights (std::vector< uint32_t > &) const

long	checkBin (const ibis::qRange &cmp, uint32_t jbin, ibis::bitvector &res) const
	Candidate check using the binned values. More...

long	checkBin (const ibis::qRange &cmp, uint32_t jbin, const ibis::bitvector &mask, ibis::bitvector &res) const
	Candidate check using the binned values. More...

void	construct (const char *)
	Construct a binned bitmap index. More...

template<typename E >
void	construct (const array_t< E > &varr)
	Construction function for in-memory data. More...

virtual int	contractRange (ibis::qContinuousRange &rng) const

virtual index *	dup () const
	Duplicate the content of an index object.

virtual uint32_t	estimate (const ibis::qContinuousRange &expr) const
	Compute an upper bound on the number of hits.

virtual void	estimate (const ibis::deprecatedJoin &expr, ibis::bitvector64 &lower, ibis::bitvector64 &upper) const
	Estimate the hits for symmetric joins. More...

virtual void	estimate (const ibis::deprecatedJoin &expr, const ibis::bitvector &mask, ibis::bitvector64 &lower, ibis::bitvector64 &upper) const

virtual void	estimate (const ibis::deprecatedJoin &expr, const ibis::bitvector &mask, const ibis::qRange const range1, const ibis::qRange const range2, ibis::bitvector64 &lower, ibis::bitvector64 &upper) const
	Evaluating a join condition with one (likely composite) index.

virtual int64_t	estimate (const ibis::deprecatedJoin &expr, const ibis::bitvector &mask, const ibis::qRange const range1, const ibis::qRange const range2) const

virtual void	estimate (const ibis::bin &idx2, const ibis::deprecatedJoin &expr, ibis::bitvector64 &lower, ibis::bitvector64 &upper) const
	Estimate the number of hits for nonsymmetric joins.

virtual void	estimate (const ibis::bin &idx2, const ibis::deprecatedJoin &expr, const ibis::bitvector &mask, ibis::bitvector64 &lower, ibis::bitvector64 &upper) const

virtual void	estimate (const ibis::bin &idx2, const ibis::deprecatedJoin &expr, const ibis::bitvector &mask, const ibis::qRange const range1, const ibis::qRange const range2, ibis::bitvector64 &lower, ibis::bitvector64 &upper) const

virtual int64_t	estimate (const ibis::bin &idx2, const ibis::deprecatedJoin &expr) const

virtual int64_t	estimate (const ibis::bin &idx2, const ibis::deprecatedJoin &expr, const ibis::bitvector &mask) const

virtual int64_t	estimate (const ibis::bin &idx2, const ibis::deprecatedJoin &expr, const ibis::bitvector &mask, const ibis::qRange const range1, const ibis::qRange const range2) const

virtual double	estimateCost (const ibis::qContinuousRange &expr) const
	Estimate the cost of evaluating a range condition.

virtual double	estimateCost (const ibis::qDiscreteRange &expr) const
	Estimate the cost of evaluating a range condition.

virtual int	expandRange (ibis::qContinuousRange &rng) const
	The functions expandRange and contractRange expands or contracts the boundaries of a range condition so that the new range will have exact answers using the function estimate. More...

virtual long	getCumulativeDistribution (std::vector< double > &bds, std::vector< uint32_t > &cts) const
	Compute the cumulative distribution from the binned index.

virtual long	getDistribution (std::vector< double > &bbs, std::vector< uint32_t > &cts) const
	Compute a histogram from the binned index.

virtual double	getMax () const
	Compute the actual maximum value from the binned index.

virtual double	getMin () const
	Compute the actual minimum value from the binned index.

virtual double	getSum () const
	Compute the approximate value of the sum from the binned index.

array_t< uint32_t > *	indices (const ibis::bitvector &mask) const

virtual void	locate (const ibis::qContinuousRange &expr, uint32_t &cand0, uint32_t &cand1) const
	Find the outer boundaries of the range expression. More...

virtual void	locate (const ibis::qContinuousRange &expr, uint32_t &cand0, uint32_t &cand1, uint32_t &hit0, uint32_t &hit1) const
	Find the bins related to the range expression. More...

virtual uint32_t	numBins () const

int	read (int fdes, size_t offset, const char fname, const char header)
	Read an ibis::bin embedded inside a file. More...

virtual long	select (const ibis::qContinuousRange &, void *) const
	Select the rows that satisfy the range condition. More...

virtual long	select (const ibis::qContinuousRange &, void *, ibis::bitvector &) const
	Select the rows that satisfy the range condition. More...

virtual void	serialSizes (uint64_t &, uint64_t &, uint64_t &) const
	Compute the size of arrays that would be generated by the serializatioin function (write). More...

virtual void	speedTest (std::ostream &out) const
	Time some logical operations and print out their speed.

virtual float	undecidable (const ibis::qContinuousRange &expr, ibis::bitvector &iffy) const
	Mark the position of the rows that can not be decided with this index. More...

virtual int	write (ibis::array_t< double > &, ibis::array_t< int64_t > &, ibis::array_t< uint32_t > &) const
	Save index to three arrays. Serialize the index in memory.

Public Member Functions inherited from ibis::index
void	addBins (uint32_t ib, uint32_t ie, ibis::bitvector &res) const
	Add the sum of `bits`[ib] through `bits`[ie-1] to `res`. More...

void	addBins (uint32_t ib, uint32_t ie, ibis::bitvector &res, const ibis::bitvector &tot) const
	Compute the sum of bit vectors [`ib`, `ie`). More...

bool	empty () const
	The index object is considered empty if there is no bitmap or getNRows returns 0. More...

virtual void	estimate (const ibis::qDiscreteRange &expr, ibis::bitvector &lower, ibis::bitvector &upper) const
	Estimate the hits for discrete ranges, i.e., those translated from 'a IN (x, y, ..)'. More...

virtual uint32_t	estimate (const ibis::qDiscreteRange &expr) const

virtual void	estimate (const ibis::index &idx2, const ibis::deprecatedJoin &expr, ibis::bitvector64 &lower, ibis::bitvector64 &upper) const
	Estimate the pairs for the range join operator.

virtual void	estimate (const ibis::index &idx2, const ibis::deprecatedJoin &expr, const ibis::bitvector &mask, ibis::bitvector64 &lower, ibis::bitvector64 &upper) const
	Estimate the pairs for the range join operator. More...

virtual void	estimate (const ibis::index &idx2, const ibis::deprecatedJoin &expr, const ibis::bitvector &mask, const ibis::qRange const range1, const ibis::qRange const range2, ibis::bitvector64 &lower, ibis::bitvector64 &upper) const

virtual int64_t	estimate (const ibis::index &idx2, const ibis::deprecatedJoin &expr) const
	Estimate an upper bound for the number of pairs.

virtual int64_t	estimate (const ibis::index &idx2, const ibis::deprecatedJoin &expr, const ibis::bitvector &mask) const
	Estimate an upper bound for the number of pairs produced from marked records. More...

virtual int64_t	estimate (const ibis::index &idx2, const ibis::deprecatedJoin &expr, const ibis::bitvector &mask, const ibis::qRange const range1, const ibis::qRange const range2) const

virtual const ibis::bitvector *	getBitvector (uint32_t i) const
	Return a pointer to the ith bitvector used in the index (may be 0).

uint32_t	getNRows () const
	Return the number of rows represented by this object.

virtual uint32_t	numBitvectors () const
	Returns the number of bit vectors used by the index.

float	sizeInBytes () const
	Estiamte the size of this index object measured in bytes. More...

void	sumBins (uint32_t ib, uint32_t ie, ibis::bitvector &res) const
	Sum up `bits[ib:ie-1]` and place the result in res. More...

void	sumBins (uint32_t ib, uint32_t ie, ibis::bitvector &res, uint32_t ib0, uint32_t ie0) const
	Compute a new sum for bit vectors [ib, ie) by taking advantage of the old sum for bitvectors [ib0, ie0). More...

void	sumBins (uint32_t ib, uint32_t ie, ibis::bitvector &res, uint32_t *buf) const
	Sum up `bits[ib:ie-1]` and place the result in res. More...

void	sumBins (const ibis::array_t< uint32_t > &, ibis::bitvector &) const
	Sum up the bits in in the specified bins.

virtual float	undecidable (const ibis::qDiscreteRange &expr, ibis::bitvector &iffy) const

virtual	~index ()
	The destructor.

Protected Member Functions
virtual void	clear ()
	Clear the existing content. More...

virtual size_t	getSerialSize () const throw ()
	Estimate the size of the serialized version of the index. More...

Protected Member Functions inherited from ibis::bin
void	addBounds (double lbd, double rbd, uint32_t nbins, uint32_t eqw)
	The function used by setBoudaries() to actually generate the bounds.

	bin (const ibis::column c, const uint32_t nbits, ibis::fileManager::storage st, size_t offset=8)
	!< The minimal values in each bin. More...

void	binning (const char *f, const std::vector< double > &bd)
	Generate bins according to the specified boundaries. More...

void	binning (const char *f, const array_t< double > &bd)

void	binning (const char *f)
	Read the data file and partition the values into bins according to the specified bin boundary. More...

template<typename E >
void	binning (const array_t< E > &varr)

template<typename E >
void	binning (const array_t< E > &varr, const array_t< double > &bd)

template<typename E >
void	binningT (const char *fname)
	Read the data file, partition the values, and write out the bin ordered data with .bin suffix. More...

long	binOrder (const char *fname) const
	Write bin-ordered values.

template<typename E >
long	binOrderT (const char *fname) const
	Write bin-ordered values.

template<typename E >
long	checkBin0 (const ibis::qRange &cmp, uint32_t jbin, ibis::bitvector &res) const

template<typename E >
long	checkBin1 (const ibis::qRange &cmp, uint32_t jbin, const ibis::bitvector &mask, ibis::bitvector &res) const

virtual double	computeSum () const
	Compute the sum of values from the information in the index. More...

void	convertGranules (granuleMap &gmap)
	Convert the granule map into binned index. More...

void	divideBitmaps (const array_t< bitvector * > &bms, std::vector< unsigned > &parts) const
	Partition the bitmaps into groups of takes about the same amount of storage. More...

virtual uint32_t	locate (const double &val) const
	Find the bin containing val. More...

template<typename E >
void	mapGranules (const array_t< E > &, granuleMap &gmap) const

template<typename T >
long	mergeValues (const ibis::qContinuousRange &, ibis::array_t< T > &) const
	Extract values only. More...

template<typename T >
long	mergeValues (const ibis::qContinuousRange &, ibis::array_t< T > &, ibis::bitvector &) const
	Extract values and record the positions. More...

void	printGranules (std::ostream &out, const granuleMap &gmap) const

void	readBinBoundaries (const char *name, uint32_t nb)
	Read a file containing a list of floating-point numbers. More...

template<typename E >
void	scanAndPartition (const array_t< E > &, unsigned)

void	scanAndPartition (const char *, unsigned, uint32_t nbins=0)
	Partition the range based on the (approximate) histogram of the data. More...

void	setBoundaries (const char *f)
	Set bin boundaries. More...

void	setBoundaries (array_t< double > &bnds, const ibis::bin &bin0) const

void	setBoundaries (array_t< double > &bnds, const ibis::bin &idx1, const array_t< uint32_t > cnt1, const array_t< uint32_t > cnt0) const

template<typename E >
void	setBoundaries (const array_t< E > &varr)

void	swap (bin &rhs)
	Swap the content of the index.

int	write32 (int fptr) const
	Write the content to a file already open.

int	write64 (int fptr) const
	write the content to a file already open.

Protected Member Functions inherited from ibis::index
virtual void	activate () const
	Regenerate all bitvectors from the underlying storage. More...

virtual void	activate (uint32_t i) const
	Regenerate the ith bitvector from the underlying storage.

virtual void	activate (uint32_t i, uint32_t j) const
	Regenerate bitvectors i (inclusive) through j (exclusive) from the underlying storage. More...

void	computeMinMax (const char *f, double &min, double &max) const

void	dataFileName (std::string &name, const char *f=0) const
	Generate data file name from "f". More...

	index (const ibis::column *c=0)
	Default constructor. More...

	index (const ibis::column c, ibis::fileManager::storage s)
	Constructor with a storage object. More...

	index (const index &)
	Copy constructor.

void	indexFileName (std::string &name, const char *f=0) const
	Generates index file name from "f". More...

void	initBitmaps (int fdes)
	Prepare the bitmaps using the given file descriptor. More...

void	initBitmaps (ibis::fileManager::storage *st)
	Prepare bitmaps from the given storage object. More...

void	initBitmaps (uint32_t *st)
	Prepare bitmaps from the given raw pointer. More...

void	initBitmaps (void *ctx, FastBitReadBitmaps rd)
	Prepare bitmaps from the user provided function pointer and context. More...

int	initOffsets (int64_t *, size_t)
	Initialize the offsets from the given data array. More...

int	initOffsets (int fdes, const char offsize, size_t start, uint32_t nobs)
	Read in the offset array. More...

int	initOffsets (ibis::fileManager::storage *st, size_t start, uint32_t nobs)
	Regenerate the offsets array from the given storage object. More...

void	mapValues (const char *f, VMap &bmap) const
	Map the positions of each individual value. More...

void	mapValues (const char *f, histogram &hist, uint32_t count=0) const
	Generate a histogram. More...

index &	operator= (const index &)
	Assignment operator.

void	optionalUnpack (array_t< ibis::bitvector * > &bits, const char *opt)
	A function to decide whether to uncompress the bitvectors. More...

Additional Inherited Members
Public Types inherited from ibis::bin
typedef std::map< double, granule * >	granuleMap

Public Types inherited from ibis::index
typedef std::map< double, uint32_t >	histogram

enum	INDEX_TYPE { BINNING =0, RANGE, MESA, AMBIT, PALE, PACK, ZONE, RELIC, ROSTER, SKIVE, FADE, SBIAD, SAPID, EGALE, MOINS, ENTRE, BAK, BAK2, KEYWORDS, MESH, BAND, DIREKTE, GENERIC, BYLT, FUZZ, ZONA, FUGE, SLICE, EXTERN }
	The integer values of this enum type are used in the index files to differentiate the indexes. More...

typedef std::map< double, ibis::bitvector * >	VMap

Static Public Member Functions inherited from ibis::bin
static uint32_t	parseNbins (const ibis::column &)
	Parse the index specs to determine eqw and nbins. More...

static unsigned	parsePrec (const ibis::column &)
	Parse the index spec to extract precision.

static unsigned	parseScale (const ibis::column &)
	Parse the specification about scaling. More...

static unsigned	parseScale (const char *)

Static Public Member Functions inherited from ibis::index
static void	addBits (const array_t< bitvector * > &bits, uint32_t ib, uint32_t ie, ibis::bitvector &res)
	Add the `pile`[ib:ie-1] to `res`. More...

static index *	create (const column c, const char name=0, const char *spec=0, int inEntirety=0)
	Index factory. More...

static void	divideCounts (array_t< uint32_t > &bounds, const array_t< uint32_t > &cnt)
	Determine how to split the array `cnt`, so that each group has roughly the same total value. More...

static bool	isIndex (const char *f, INDEX_TYPE t)
	Is the named file an index file? Read the header of the named file to determine if it contains an index of the specified type. More...

template<typename E >
static void	mapValues (const array_t< E > &val, VMap &bmap)

template<typename E >
static void	mapValues (const array_t< E > &val, histogram &hist, uint32_t count=0)

template<typename E >
static void	mapValues (const array_t< E > &val, array_t< E > &bounds, std::vector< uint32_t > &cnts)

template<typename E1 , typename E2 >
static void	mapValues (const array_t< E1 > &val1, const array_t< E2 > &val2, array_t< E1 > &bnd1, array_t< E2 > &bnd2, std::vector< uint32_t > &cnts)
	Compute a two-dimensional histogram. More...

static void	printHeader (std::ostream &, const char *)

static void	setBases (array_t< uint32_t > &bases, uint32_t card, uint32_t nbase=2)
	Fill the array bases with the values that cover the range [0, card). More...

static void	sumBits (const array_t< bitvector * > &bits, uint32_t ib, uint32_t ie, ibis::bitvector &res)
	Sum up `pile`[ib:ie-1] and place the result in `res`. More...

static void	sumBits (const array_t< bitvector * > &bits, const ibis::bitvector &tot, uint32_t ib, uint32_t ie, ibis::bitvector &res)
	Sum up `pile`[ib:ie-1] and add the result to `res`. More...

Static Protected Member Functions inherited from ibis::index
static void	indexFileName (std::string &name, const ibis::column col1, const ibis::column col2, const char *f=0)
	Generate the index file name for the composite index fromed on two columns. More...

Protected Attributes inherited from ibis::bin
array_t< double >	bounds
	!< Number of bitvectors.

array_t< double >	maxval
	!< The nominal boundaries.

array_t< double >	minval
	!< The maximal values in each bin.

uint32_t	nobs

Protected Attributes inherited from ibis::index
array_t< ibis::bitvector * >	bits
	A list of bitvectors.

bitmapReader *	breader
	The functor to read serialized bitmaps from a more complex source.

const ibis::column *	col
	Pointer to the column this index is for.

const char *	fname
	The name of the file containing the index.

uint32_t	nrows
	The number of rows represented by the index. More...

array_t< int32_t >	offset32
	Starting positions of the bitvectors.

array_t< int64_t >	offset64
	Starting positions of the bitvectors. More...

ibis::fileManager::storage *	str
	The underlying storage. More...

Detailed Description

A two-level index.

The top (coarse) level uses the interval encoding and the bottom (fine) level uses the equality encoding. Similar to class ibis::fuzz, the fine level bitmaps are kept together as in ibis::bin and the coarse level bitmaps are placed at the end of the index file.

Constructor & Destructor Documentation

ibis::fuge::fuge	(	const ibis::column *	c,
		ibis::fileManager::storage *	st,
		size_t	start = `8`
	)

Reconstruct from content of a file.

The leading portion of the index file is the same as ibis::bin, which allows the constructor of the base class to work properly. The content following the last bitvector in ibis::bin is as follows,

See also: ibis::fuge::writeCoarse.
nc (uint32_t) -- number of coarse bins.

cbounds (unsigned[nc+1]) -- boundaries of the coarse bins.

coffsets([nc-ceil(nc/2)+2]) -- starting positions (32/64-bit).

cbits (bitvector[nc-ceil(nc/2)+1]) -- bitvectors.

References ibis::fileManager::storage::begin(), ibis::array_t< T >::clear(), clear(), ibis::index::col, ibis::fileManager::storage::isFileMap(), ibis::part::name(), ibis::column::name(), ibis::index::nrows, ibis::index::offset32, ibis::index::offset64, print(), ibis::fileManager::storage::size(), and ibis::array_t< T >::swap().

Member Function Documentation

void ibis::fuge::adjustLength ( uint32_t nr )

virtual

Fill the bitmaps to the specified size.

Fill the bitvectors with zeros so that they all contain nrows bits.

Truncate the bitvectors if they have more bits.

Reimplemented from ibis::bin.

References ibis::bin::adjustLength().

virtual void ibis::fuge::clear ( )

inlineprotectedvirtual

Clear the existing content.

Free the bitmap objectes common to all index objects.

Reimplemented from ibis::bin.

References ibis::bin::clear().

Referenced by fuge().

void ibis::fuge::estimate	(	const ibis::qContinuousRange &	expr,
		ibis::bitvector &	lower,
		ibis::bitvector &	upper
	)		const

virtual

Provide an estimation based on the current index.

Set bits in lower are hits for certain, set bits in upper are candidates. Set bits in (upper - lower) should be checked to verifies which ones are actually hits. If the bitvector upper contain less bits than bitvector lower, the content of upper is assumed to be the same as lower.

Note: This function will not do anything if the estimated cost is high.

Reimplemented from ibis::bin.

References ibis::bitvector::clear(), ibis::bitvector::copy(), and ibis::bitvector::set().

long ibis::fuge::evaluate	(	const ibis::qContinuousRange &	expr,
		ibis::bitvector &	hits
	)		const

virtual

To evaluate the exact hits.

On success, return the number of hits, otherwise a negative value is returned.

Reimplemented from ibis::bin.

References ibis::bitvector::cnt(), and ibis::bitvector::size().

virtual long ibis::fuge::evaluate	(	const ibis::qDiscreteRange &	,
		ibis::bitvector &
	)		const

inlinevirtual

To evaluate the exact hits.

On success, return the number of hits, otherwise a negative value is returned.

Reimplemented from ibis::bin.

References ibis::index::evaluate().

size_t ibis::fuge::getSerialSize	(	)	const
throw	(
	)

protectedvirtual

Estimate the size of the serialized version of the index.

Return the size in bytes.

Reimplemented from ibis::bin.

References ibis::bitvector::size().

virtual const char* ibis::fuge::name ( ) const

inlinevirtual

Returns the name of the index, similar to the function type, but returns a string instead.

Reimplemented from ibis::bin.

void ibis::fuge::print ( std::ostream & out ) const

virtual

Prints human readable information.

Outputs information about the index as text to the specified output stream.

Reimplemented from ibis::bin.

Referenced by fuge().

The documentation for this class was generated from the following files:

ibin.h
ixfuge.cpp



	Contact us Disclaimers FastBit source code FastBit mailing list archive

Public Member Functions

Protected Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation