ikeywords.h

Go to the documentation of this file.

//File: $Id$
// Author: John Wu <John.Wu at ACM.org>
// Copyright (c) 2006-2016 the Regents of the University of California
#ifndef IBIS_KEYWORDS_H
#define IBIS_KEYWORDS_H

#include "index.h"      // base index class
#include "category.h"   // definitions of string-valued columns

class ibis::keywords : public ibis::index {
public:
    virtual ~keywords() {clear();}
    explicit keywords(const ibis::column* c, const char* f=0);
    keywords(const ibis::column* c, ibis::text::tokenizer& tkn,
             const char* f=0);
    keywords(const ibis::column* c, ibis::fileManager::storage* st);

    virtual INDEX_TYPE type() const {return KEYWORDS;}
    virtual const char* name() const {return "keywords";}
    virtual void binBoundaries(std::vector<double>& b) const {b.clear();}
    virtual void binWeights(std::vector<uint32_t>& b) const;
    virtual double getMin() const {return DBL_MAX;}
    virtual double getMax() const {return -DBL_MAX;}
    virtual double getSum() const {return -DBL_MAX;}
    long search(const char*, ibis::bitvector&) const;
    long search(const char*) const;
    long search(const std::vector<std::string>&, ibis::bitvector&) const;
    long search(const std::vector<std::string>&) const;

    virtual index* dup() const;
    virtual void print(std::ostream& out) const;
    virtual void serialSizes(uint64_t&, uint64_t&, uint64_t&) const;
    virtual int write(ibis::array_t<double> &,
                      ibis::array_t<int64_t> &,
                      ibis::array_t<uint32_t> &) const;
    virtual int write(const char* dt) const;
    virtual int read(const char* idxfile);
    virtual int read(ibis::fileManager::storage* st);
    virtual long append(const char* dt, const char* df, uint32_t nnew);

    using ibis::index::evaluate;
    using ibis::index::estimate;
    using ibis::index::undecidable;
    virtual long evaluate(const ibis::qContinuousRange& expr,
                          ibis::bitvector& hits) const;
    virtual void estimate(const ibis::qContinuousRange& expr,
                          ibis::bitvector& lower,
                          ibis::bitvector& upper) const;
    virtual uint32_t estimate(const ibis::qContinuousRange& expr) const;
    virtual float undecidable(const ibis::qContinuousRange &,
                              ibis::bitvector &iffy) const {
        iffy.clear();
        return 0.0;
    }
    virtual double estimateCost(const ibis::qContinuousRange& expr) const;
    virtual double estimateCost(const ibis::qDiscreteRange& expr) const;

    virtual long select(const ibis::qContinuousRange&, void*) const {
        return -1;}
    virtual long select(const ibis::qContinuousRange&, void*,
                        ibis::bitvector&) const {
        return -1;}

    class tokenizer;

protected:
    ibis::dictionary terms;

    virtual size_t getSerialSize() const throw();
    int readTermDocFile(const ibis::column* idcol, const char* f);
    inline char readTerm(const char*& buf, std::string &key) const;
    inline uint32_t readUInt(const char*& buf) const;
    int readTDLine(std::istream& in, std::string& key,
                   std::vector<uint32_t>& idlist,
                   char* buf, uint32_t nbuf) const;
    void setBits(std::vector<uint32_t>& pos, ibis::bitvector& bvec) const;
    int parseTextFile(ibis::text::tokenizer &tkn, const char *f);

    void clear();
    void reorderTerms();
}; // class ibis::keywords

inline char ibis::keywords::readTerm(const char*& buf,
                                     std::string &keyword) const {
    while (isspace(*buf)) // skip leading space
        ++ buf;
    while (isprint(*buf)) { // loop through all printable till the delimiter
        if (*buf == ':') {
            return *buf;
        }
        else if (isspace(*buf)) {
            for (++ buf; isspace(*buf); ++ buf);
            if (*buf == ':') {
                return *buf;
            }
            else {
                keyword += ' ';
                keyword += *buf;
                ++ buf;
            }
        }
        else {
            keyword += *buf;
            ++ buf;
        }
    }
    return *buf;
} // ibis::keywords::readTerm

inline uint32_t ibis::keywords::readUInt(const char*& buf) const {
    uint32_t res = 0;
    while (*buf && ! isdigit(*buf)) // skip leading non-digit
        ++ buf;

    while (isdigit(*buf)) {
        res = res * 10 + (*buf - '0');
        ++ buf;
    }
    return res;
} // ibis::keywords::readUInt

class ibis::keywords::tokenizer : public ibis::text::tokenizer {
public:
    tokenizer(const char *d=ibis::util::delimiters);
    virtual ~tokenizer() {}

    virtual int operator()(std::vector<const char*>& tkns, char *buf);

protected:
    std::string delim_;
}; // class ibis::keywords::tokenizer
#endif