column.h

Go to the documentation of this file.

//File: $Id$
// Author: John Wu <John.Wu at ACM.org>
// Copyright (c) 2000-2016 the Regents of the University of California
#ifndef IBIS_COLUMN_H
#define IBIS_COLUMN_H
#include "table.h"      // ibis::TYPE_T
#include "qExpr.h"      // ibis::qContinuousRange
#include "bitvector.h"
#include <string>

namespace ibis { // additional names to the namespace ibis
    // derived classes of ibis::column, implemented in category.cpp
    class category;   // for categorical values (low-cardinality text fields)
    class text;       // arbitrary cardinality text fields
    class blob;       // text may contain null characters
    class collis;     // data accessed through FastBitReadExtArray

    // the following are used for storing selected values of different types
    // of columns (implemented in colValues.cpp)
    class colBytes;
    class colUBytes;
    class colShorts;
    class colUShorts;
    class colInts;
    class colUInts;
    class colLongs;
    class colULongs;
    class colFloats;
    class colDoubles;
    class colStrings;
    class colBlobs;
} // namespace

class FASTBIT_CXX_DLLSPEC ibis::column {
public:

    virtual ~column();
    column(const column& rhs);
    column(const part* tbl, FILE* file);
    column(const part* tbl, ibis::TYPE_T t, const char* name,
           const char* desc="", double low=DBL_MAX, double high=-DBL_MAX);

    ibis::TYPE_T type() const {return m_type;}
    const char* name() const {return m_name.c_str();}
    void name(const char* nm) {m_name = nm;}
    const char* description() const {return m_desc.c_str();}
    std::string fullname() const;
    const double& lowerBound() const {return lower;}
    const double& upperBound() const {return upper;}

    bool isFloat() const;
    bool isInteger() const;
    bool isSignedInteger() const;
    bool isUnsignedInteger() const;
    bool isNumeric() const;
    bool isSorted() const {return m_sorted;} 
    bool hasIndex() const;
    bool hasRoster() const;

    void description(const char* d) {m_desc = d;}
    void lowerBound(double d) {lower = d;}
    void upperBound(double d) {upper = d;}
    void isSorted(bool);
    int  elementSize() const;
    int  nRows() const {return mask_.size();}

    const part*  partition() const {return thePart;}
    const part*& partition() {return thePart;}

    // function related to index/bin
    const char* indexSpec() const; 
    uint32_t numBins() const; 
    void indexSpec(const char* spec) {m_bins=spec;}
    void preferredBounds(std::vector<double>&) const;
    void binWeights(std::vector<uint32_t>&) const;

    virtual void computeMinMax();
    virtual void computeMinMax(const char *dir);
    virtual void computeMinMax(const char *dir,
                               double& min, double &max, bool &asc) const;

    virtual int  attachIndex(double *, uint64_t, int64_t *, uint64_t,
                             void *, FastBitReadBitmaps) const;
    virtual int  attachIndex(double *, uint64_t, int64_t *, uint64_t,
                             uint32_t *, uint64_t) const;
    virtual void loadIndex(const char* iopt=0, int ropt=0) const throw ();
    virtual void unloadIndex() const;
    virtual long indexSize() const;

    uint32_t indexedRows() const;
    void indexSpeedTest() const;
    void purgeIndexFile(const char *dir=0) const;

    const char* dataFileName(std::string& fname, const char *dir=0) const;
    const char* nullMaskName(std::string& fname) const;
    void getNullMask(bitvector& mask) const;
    int  setNullMask(const bitvector&);

    virtual const char* findString(const char*) const
    {return static_cast<const char*>(0);}
    virtual int getString(uint32_t, std::string&) const {return -1;}
    virtual int getOpaque(uint32_t, ibis::opaque&) const;

    array_t<int32_t>* getIntArray() const;
    array_t<float>*   getFloatArray() const;
    array_t<double>*  getDoubleArray() const;
    virtual int getValuesArray(void* vals) const;
    virtual ibis::fileManager::storage* getRawData() const;
    virtual bool hasRawData() const;
    int  getDataflag() const {return dataflag;}
    void setDataflag(int df) {dataflag = df;}

    virtual array_t<signed char>*   selectBytes(const bitvector& mask) const;
    virtual array_t<unsigned char>* selectUBytes(const bitvector& mask) const;
    virtual array_t<int16_t>*  selectShorts(const bitvector& mask) const;
    virtual array_t<uint16_t>* selectUShorts(const bitvector& mask) const;
    virtual array_t<int32_t>*  selectInts(const bitvector& mask) const;
    virtual array_t<uint32_t>* selectUInts(const bitvector& mask) const;
    virtual array_t<int64_t>*  selectLongs(const bitvector& mask) const;
    virtual array_t<uint64_t>* selectULongs(const bitvector& mask) const;
    virtual array_t<float>*    selectFloats(const bitvector& mask) const;
    virtual array_t<double>*   selectDoubles(const bitvector& mask) const;
    virtual std::vector<std::string>*
        selectStrings(const bitvector& mask) const;
    virtual std::vector<ibis::opaque>*
        selectOpaques(const bitvector& mask) const;

    long selectValues(const bitvector&, void*) const;
    long selectValues(const bitvector&, void*, array_t<uint32_t>&) const;
    long selectValues(const ibis::qContinuousRange&, void*) const;

    virtual void write(FILE* file) const;
    virtual void print(std::ostream& out) const;
    void logMessage(const char* event, const char* fmt, ...) const;
    void logWarning(const char* event, const char* fmt, ...) const;

    int expandRange(ibis::qContinuousRange& rng) const;
    int contractRange(ibis::qContinuousRange& rng) const;

    virtual long evaluateRange(const ibis::qContinuousRange& cmp,
                               const ibis::bitvector& mask,
                               ibis::bitvector& res) const;
    virtual long evaluateRange(const ibis::qDiscreteRange& cmp,
                               const ibis::bitvector& mask,
                               ibis::bitvector& res) const;
    virtual long evaluateRange(const ibis::qIntHod& cmp,
                               const ibis::bitvector& mask,
                               ibis::bitvector& res) const;
    virtual long evaluateRange(const ibis::qUIntHod& cmp,
                               const ibis::bitvector& mask,
                               ibis::bitvector& res) const;

    virtual long stringSearch(const char*, ibis::bitvector&) const;
    virtual long stringSearch(const std::vector<std::string>&,
                              ibis::bitvector&) const;
    virtual long stringSearch(const char*) const;
    virtual long stringSearch(const std::vector<std::string>&) const;
    virtual long keywordSearch(const char*, ibis::bitvector&) const;
    virtual long keywordSearch(const char*) const;
    virtual long keywordSearch(const std::vector<std::string>&,
                               ibis::bitvector&) const;
    virtual long keywordSearch(const std::vector<std::string>&) const;
    virtual long patternSearch(const char*) const;
    virtual long patternSearch(const char*, ibis::bitvector &) const;

    virtual long evaluateAndSelect(const ibis::qContinuousRange&,
                                   const ibis::bitvector&, void*,
                                   ibis::bitvector&) const;

    virtual long estimateRange(const ibis::qContinuousRange& cmp,
                               ibis::bitvector& low,
                               ibis::bitvector& high) const;
    virtual long estimateRange(const ibis::qDiscreteRange& cmp,
                               ibis::bitvector& low,
                               ibis::bitvector& high) const;
    virtual long estimateRange(const ibis::qIntHod& cmp,
                               ibis::bitvector& low,
                               ibis::bitvector& high) const;
    virtual long estimateRange(const ibis::qUIntHod& cmp,
                               ibis::bitvector& low,
                               ibis::bitvector& high) const;

    virtual long estimateRange(const ibis::qContinuousRange& cmp) const;
    virtual long estimateRange(const ibis::qDiscreteRange& cmp) const;
    virtual long estimateRange(const ibis::qIntHod& cmp) const;
    virtual long estimateRange(const ibis::qUIntHod& cmp) const;

    virtual double estimateCost(const ibis::qContinuousRange& cmp) const;
    virtual double estimateCost(const ibis::qDiscreteRange& cmp) const;
    virtual double estimateCost(const ibis::qIntHod& cmp) const;
    virtual double estimateCost(const ibis::qUIntHod& cmp) const;
    virtual double estimateCost(const ibis::qString&) const {
        return 0;}
    virtual double estimateCost(const ibis::qAnyString&) const {
        return 0;}

    virtual float getUndecidable(const ibis::qContinuousRange& cmp,
                                 ibis::bitvector& iffy) const;
    virtual float getUndecidable(const ibis::qDiscreteRange& cmp,
                                 ibis::bitvector& iffy) const;
    virtual float getUndecidable(const ibis::qIntHod& cmp,
                                 ibis::bitvector& iffy) const;
    virtual float getUndecidable(const ibis::qUIntHod& cmp,
                                 ibis::bitvector& iffy) const;

    virtual const ibis::dictionary* getDictionary() const {return 0;}

    virtual long append(const char* dt, const char* df, const uint32_t nold,
                        const uint32_t nnew, uint32_t nbuf, char* buf);

    virtual long append(const void* vals, const ibis::bitvector& msk);
    virtual long writeData(const char* dir, uint32_t nold, uint32_t nnew,
                           ibis::bitvector& mask, const void *va1,
                           void *va2=0);
    template <typename T>
    long castAndWrite(const array_t<double>& vals, ibis::bitvector& mask,
                      const T special);
    virtual long saveSelected(const ibis::bitvector& sel, const char *dest,
                              char *buf, uint32_t nbuf);
    virtual long truncateData(const char* dir, uint32_t nent,
                              ibis::bitvector& mask) const;

    virtual int indexWrite(ibis::array_t<double> &,
                           ibis::array_t<int64_t> &,
                           ibis::array_t<uint32_t> &) const;
    virtual void indexSerialSizes(uint64_t&, uint64_t&, uint64_t&) const;

    virtual double getActualMin() const;
    virtual double getActualMax() const;
    virtual double getSum() const;
    long getCumulativeDistribution(std::vector<double>& bounds,
                                   std::vector<uint32_t>& counts) const;
    long getDistribution(std::vector<double>& bbs,
                         std::vector<uint32_t>& counts) const;
    class info;
    class indexLock;
    class mutexLock;

    struct unixTimeScribe {
        ~unixTimeScribe() {
            delete format_;
            delete timezone_;
        }
        unixTimeScribe(const char *fmt, const char *tz=0)
            : format_(ibis::util::strnewdup(fmt)),
              timezone_(ibis::util::strnewdup(tz)) {}
        unixTimeScribe(const unixTimeScribe &rhs)
            : format_(ibis::util::strnewdup(rhs.format_)),
              timezone_(ibis::util::strnewdup(rhs.timezone_)) {}

        unixTimeScribe& operator=(const unixTimeScribe &rhs) {
            delete format_;
            delete timezone_;
            format_ = ibis::util::strnewdup(rhs.format_);
            timezone_ = ibis::util::strnewdup(rhs.timezone_);
            return *this;
        }

        unixTimeScribe* dup() const {
            return new unixTimeScribe(*this);
        }

        void operator()(std::ostream&, int64_t) const;
        void operator()(std::ostream&, double) const;

        const char *format_;
        const char *timezone_;
    }; // unixTimeScribe
    void setTimeFormat(const char*);
    void setTimeFormat(const unixTimeScribe &);
    const unixTimeScribe* getTimeFormat() const {return m_utscribe;}

    template <typename T> static
    void actualMinMax(const array_t<T>& vals, const ibis::bitvector& mask,
                      double& min, double& max, bool &asc);
    template <typename T> static
    T computeMin(const array_t<T>& vals, const ibis::bitvector& mask);
    template <typename T> static
    T computeMax(const array_t<T>& vals, const ibis::bitvector& mask);
    template <typename T> static
    double computeSum(const array_t<T>& vals, const ibis::bitvector& mask);

protected:
    // protected member variables
    const part* thePart;  
    ibis::bitvector mask_;
    ibis::TYPE_T m_type;  
    std::string m_name;   
    std::string m_desc;   
    std::string m_bins;   
    bool m_sorted;        
    double lower;         
    double upper;         
    unixTimeScribe *m_utscribe;
    mutable int dataflag;
    mutable ibis::index* idx;
    mutable ibis::util::sharedInt32 idxcnt;

    void logError(const char* event, const char* fmt, ...) const;
    long string2int(int fptr, dictionary& dic, uint32_t nbuf, char* buf,
                    array_t<uint32_t>& out) const;
    double computeMin() const;
    double computeMax() const;
    double computeSum() const;
    void actualMinMax(const char *fname, const ibis::bitvector& mask,
                      double &min, double &max, bool &asc) const;

    virtual int searchSorted(const ibis::qContinuousRange&,
                             ibis::bitvector&) const;
    virtual int searchSorted(const ibis::qDiscreteRange&,
                             ibis::bitvector&) const;
    virtual int searchSorted(const ibis::qIntHod&,
                             ibis::bitvector&) const;
    virtual int searchSorted(const ibis::qUIntHod&,
                             ibis::bitvector&) const;
    template <typename T> int
        searchSortedICC(const array_t<T>& vals,
                        const ibis::qContinuousRange& rng,
                        ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedICD(const array_t<T>& vals,
                        const ibis::qDiscreteRange& rng,
                        ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedICD(const array_t<T>& vals,
                        const ibis::qIntHod& rng,
                        ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedICD(const array_t<T>& vals,
                        const ibis::qUIntHod& rng,
                        ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedOOCC(const char* fname,
                         const ibis::qContinuousRange& rng,
                         ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedOOCD(const char* fname,
                         const ibis::qDiscreteRange& rng,
                         ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedOOCD(const char* fname,
                         const ibis::qIntHod& rng,
                         ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedOOCD(const char* fname,
                         const ibis::qUIntHod& rng,
                         ibis::bitvector& hits) const;

    template <typename T> uint32_t
        findLower(int fdes, const uint32_t nr, const T tgt) const;
    template <typename T> uint32_t
        findUpper(int fdes, const uint32_t nr, const T tgt) const;

    template <typename T>
        long selectValuesT(const char*, const bitvector&, array_t<T>&) const;
    template <typename T>
        long selectValuesT(const char*, const bitvector& mask,
                           array_t<T>& vals, array_t<uint32_t>& inds) const;
    template <typename T>
        long selectToStrings(const char*, const bitvector&,
                             std::vector<std::string>&) const;
    template <typename T>
        long selectToOpaques(const char*, const bitvector&,
                             std::vector<ibis::opaque>&) const;

    template <typename T>
        long appendValues(const array_t<T>&, const ibis::bitvector&);
    long appendStrings(const std::vector<std::string>&, const ibis::bitvector&);

    class readLock;
    class writeLock;
    class softWriteLock;
    friend class readLock;
    friend class writeLock;
    friend class indexLock;
    friend class mutexLock;
    friend class softWriteLock;

private:
    mutable pthread_rwlock_t rwlock;
    mutable pthread_mutex_t mutex;

    column& operator=(const column&); // no assignment
}; // ibis::column

class FASTBIT_CXX_DLLSPEC ibis::column::info {
 public:
    const char* name;           
    const char* description;    
    const double expectedMin;   
    const double expectedMax;   
    const ibis::TYPE_T type;    
    info(const ibis::column& col);
    info(const info& rhs)
        : name(rhs.name), description(rhs.description),
        expectedMin(rhs.expectedMin),
        expectedMax(rhs.expectedMax),
        type(rhs.type) {};

 private:
    info();
    info& operator=(const info&);
}; // ibis::column::info

class ibis::column::indexLock {
public:
    ~indexLock();
    indexLock(const ibis::column* col, const char* m);
    const ibis::index* getIndex() const {return theColumn->idx;};

private:
    const ibis::column* theColumn;
    const char* mesg;

    indexLock();
    indexLock(const indexLock&);
    indexLock& operator=(const indexLock&);
}; // ibis::column::indexLock

class ibis::column::mutexLock {
public:
    mutexLock(const ibis::column* col, const char* m)
        : theColumn(col), mesg(m) {
        LOGGER(ibis::gVerbose > 9)
            << "column[" << (theColumn ? theColumn->fullname() : "?.?")
            << "]::gainExclusiveAccess for " << (mesg && *mesg ? mesg : "???");
        pthread_mutex_t *mtx = (theColumn ? &theColumn->mutex :
                                &ibis::util::envLock);
        int ierr = pthread_mutex_lock(mtx);
        LOGGER(0 != ierr && ibis::gVerbose > 0)
            << "Warning -- column["
            << (theColumn ? theColumn->fullname() : "?.?")
            << "]::gainExclusiveAccess -- pthread_mutex_lock for "
            << (mesg && *mesg ? mesg : "???") << "returned " << ierr
            << " (" << strerror(ierr) << ")";
    }
    ~mutexLock() {
        LOGGER(ibis::gVerbose > 9)
            << "column[" << (theColumn ? theColumn->fullname() : "?.?")
            << "]::releaseExclusiveAccess for "
            << (mesg && *mesg ? mesg : "???");
        pthread_mutex_t *mtx = (theColumn ? &theColumn->mutex :
                                &ibis::util::envLock);
        int ierr = pthread_mutex_unlock(mtx);
        LOGGER(0 != ierr && ibis::gVerbose > 0)
            << "Warning -- column["
            << (theColumn ? theColumn->fullname() : "?.?")
            << "]::releaseExclusiveAccess -- pthread_mutex_unlock for "
            << (mesg && *mesg ? mesg : "???") << "returned " << ierr
            << " (" << strerror(ierr) << ")";
    }

private:
    const ibis::column* theColumn;
    const char* mesg;

    mutexLock() {}; // no default constructor
    mutexLock(const mutexLock&) {}; // can not copy
    mutexLock& operator=(const mutexLock&);
}; // ibis::column::mutexLock

class ibis::column::writeLock {
public:
    writeLock(const ibis::column* col, const char* m);
    ~writeLock();

private:
    const ibis::column* theColumn;
    const char* mesg;

    writeLock();
    writeLock(const writeLock&);
    writeLock& operator=(const writeLock&);
}; // ibis::column::writeLock

class ibis::column::softWriteLock {
public:
    softWriteLock(const ibis::column* col, const char* m);
    ~softWriteLock();
    bool isLocked() const {return(locked==0);}

private:
    const ibis::column* theColumn;
    const char* mesg;
    const int locked;

    softWriteLock();
    softWriteLock(const softWriteLock&);
    softWriteLock& operator=(const softWriteLock&);
}; // ibis::column::softWriteLock

class ibis::column::readLock {
public:
    readLock(const ibis::column* col, const char* m);
    ~readLock();

private:
    const ibis::column* theColumn;
    const char* mesg;

    readLock();
    readLock(const readLock&);
    readLock& operator=(const readLock&);
}; // ibis::column::readLock

inline int ibis::column::elementSize() const {
    int sz;
    switch (m_type) {
    case ibis::OID: sz = sizeof(rid_t); break;
    case ibis::INT: sz = sizeof(int32_t); break;
    case ibis::UINT: sz = sizeof(uint32_t); break;
    case ibis::LONG: sz = sizeof(int64_t); break;
    case ibis::ULONG: sz = sizeof(uint64_t); break;
    case ibis::FLOAT: sz = sizeof(float); break;
    case ibis::DOUBLE: sz = sizeof(double); break;
    case ibis::BYTE: sz = sizeof(char); break;
    case ibis::UBYTE: sz = sizeof(unsigned char); break;
    case ibis::SHORT: sz = sizeof(int16_t); break;
    case ibis::USHORT: sz = sizeof(uint16_t); break;
    // case ibis::CATEGORY: sz = 0; break; // no fixed size per element
    // case ibis::TEXT: sz = 0; break; // no fixed size per element
    // case ibis::BLOB: sz = 0; break; // no fixed size per element
    default: sz = 0; break;
    }
    return sz;
} // ibis::column::elementSize

inline bool ibis::column::isFloat() const {
    return(m_type == ibis::FLOAT || m_type == ibis::DOUBLE);
} // ibis::column::isFloat

inline bool ibis::column::isInteger() const {
    return(m_type == ibis::BYTE || m_type == ibis::UBYTE ||
           m_type == ibis::SHORT || m_type == ibis::USHORT ||
           m_type == ibis::INT || m_type == ibis::UINT ||
           m_type == ibis::LONG || m_type == ibis::ULONG);
} // ibis::column::isInteger

inline bool ibis::column::isSignedInteger() const {
    return(m_type == ibis::BYTE || m_type == ibis::SHORT ||
           m_type == ibis::INT || m_type == ibis::LONG);
} // ibis::column::isSignedInteger

inline bool ibis::column::isUnsignedInteger() const {
    return(m_type == ibis::UBYTE || m_type == ibis::USHORT ||
           m_type == ibis::UINT || m_type == ibis::ULONG);
} // ibis::column::isUnsignedInteger

inline bool ibis::column::isNumeric() const {
    return(m_type == ibis::BYTE || m_type == ibis::UBYTE ||
           m_type == ibis::SHORT || m_type == ibis::USHORT ||
           m_type == ibis::INT || m_type == ibis::UINT ||
           m_type == ibis::LONG || m_type == ibis::ULONG ||
           m_type == ibis::FLOAT || m_type == ibis::DOUBLE);
} // ibis::column::isNumeric

// the operator to print a column to an output stream
inline std::ostream& operator<<(std::ostream& out, const ibis::column& prop) {
    prop.print(out);
    return out;
}

namespace ibis { // for template specialization
    template <> long column::selectToStrings<signed char>
    (const char*, const bitvector&, std::vector<std::string>&) const;
    template <> long column::selectToStrings<unsigned char>
    (const char*, const bitvector&, std::vector<std::string>&) const;

    namespace util {
        inline bool isFloatType(ibis::TYPE_T t) {
            return(t == ibis::FLOAT || t == ibis::DOUBLE);
        }

        inline bool isIntegerType(ibis::TYPE_T t) {
            return(t == ibis::BYTE  || t == ibis::UBYTE ||
                   t == ibis::SHORT || t == ibis::USHORT ||
                   t == ibis::INT   || t == ibis::UINT ||
                   t == ibis::LONG  || t == ibis::ULONG);
        }

        inline bool isSignedIntegerType(ibis::TYPE_T t) {
            return(t == ibis::BYTE || t == ibis::SHORT ||
                   t == ibis::INT  || t == ibis::LONG);
        }

        inline bool isUnsignedIntegerType(ibis::TYPE_T t) {
            return(t == ibis::UBYTE || t == ibis::USHORT ||
                   t == ibis::UINT  || t == ibis::ULONG);
        }

        inline bool isNumericType(ibis::TYPE_T t) {
            return(t == ibis::BYTE  || t == ibis::UBYTE ||
                   t == ibis::SHORT || t == ibis::USHORT ||
                   t == ibis::INT   || t == ibis::UINT ||
                   t == ibis::LONG  || t == ibis::ULONG ||
                   t == ibis::FLOAT || t == ibis::DOUBLE);
        }

        inline bool isStringType(ibis::TYPE_T t) {
            return(t == ibis::TEXT || t == ibis::CATEGORY);
        }
    }
}
#endif // IBIS_COLUMN_H