fileManager.h
Go to the documentation of this file.
1 // File: $Id$
2 // Author: K. John Wu <John.Wu at acm.org>
3 // Lawrence Berkeley National Laboratory
4 // Copyright (c) 2000-2016 University of California
5 #ifndef IBIS_FILEMANAGER_H
6 #define IBIS_FILEMANAGER_H
7 #include "util.h"
16 
17 #include <set> // std::set
18 #include <map> // std::map
19 
23 class FASTBIT_CXX_DLLSPEC ibis::fileManager {
24 public:
25 
30  MMAP_LARGE_FILES, // files > minMapSize are mapped if possible
31  PREFER_READ, // read the whole file into memory
32  PREFER_MMAP // try to use mmap if possible
33  };
34 
35  template<typename T>
36  int getFile(const char* name, array_t<T>& arr,
37  ACCESS_PREFERENCE pref=MMAP_LARGE_FILES);
38  template<typename T>
39  int tryGetFile(const char* name, array_t<T>& arr,
40  ACCESS_PREFERENCE pref=MMAP_LARGE_FILES);
41 
43  void printStatus(std::ostream& out) const;
45  void flushFile(const char* name);
47  void flushDir(const char* name);
49  void clear();
50 
52  static fileManager& instance();
54  static time_t iBeat() {return hbeat++;}
56  const double& pageCount() const {return page_count;}
58  static uint32_t pageSize() {return pagesize;}
59  inline void recordPages(off_t start, off_t stop);
60  static inline void increaseUse(size_t inc, const char* evt);
61  static inline void decreaseUse(size_t dec, const char* evt);
63  void signalMemoryAvailable() const;
64 
66  class cleaner {
67  public:
68  virtual void operator()() const = 0;
69  virtual ~cleaner() {};
70  };
71  void addCleaner(const cleaner* cl);
72  void removeCleaner(const cleaner* cl);
73 
74  class roFile; // forward declaration of fileManager::roFile
75  class storage; // forward declaration of fileManager::storage
76 #if defined(HAVE_FILE_MAP)
77  class rofSegment; // forward declaration of fileManager::rofSegment
78 #endif
79  friend class roFile;
80  friend class storage;
81  int getFile(const char* name, storage** st,
82  ACCESS_PREFERENCE pref=MMAP_LARGE_FILES);
83  int tryGetFile(const char* name, storage** st,
84  ACCESS_PREFERENCE pref=MMAP_LARGE_FILES);
85  static storage* getFileSegment(const char* name, const int fdes,
86  const off_t b, const off_t e);
87 
88  // /// Obtain a read lock on the file manager.
89  // inline void gainReadAccess(const char* mesg) const;
90  // /// Release a read lock on the file manager.
91  // inline void releaseAccess(const char* mesg) const;
93  // /// management of the file manager should hold a readLock.
94  // class readLock {
95  // public:
96  // /// Constructor. Acquires a read lock.
97  // readLock(const char* m) : mesg(m) {
98  // ibis::fileManager::instance().gainReadAccess(m);
99  // }
100  // /// Destructor. Releases the read lock.
101  // ~readLock() {
102  // ibis::fileManager::instance().releaseAccess(mesg);
103  // }
104  // private:
105  // /// A free-form message. Typically used to identify the holder of
106  // /// the lock.
107  // const char* mesg;
108  // };
109 
111  static uint64_t currentCacheSize() {return maxBytes;}
113  static int adjustCacheSize(uint64_t);
115  static uint64_t bytesInUse() {return ibis::fileManager::totalBytes();}
117  static uint64_t bytesFree();
118 
120  unsigned int getMaxOpenMmapFiles() const;
122  uint64_t getMaxMmapBytes() const;
123 
127  template <typename T>
128  class buffer {
129  public:
131  buffer(size_t sz=0);
133  ~buffer();
134 
136  T& operator[](size_t i) {return buf[i];}
138  const T& operator[](size_t i) const {return buf[i];}
140  T* address() const {return buf;}
142  size_t size() const {return nbuf;}
144  size_t resize(size_t sz=0);
146  void swap(buffer<T>& other) throw () {
147  T* btmp = buf;
148  buf = other.buf;
149  other.buf = btmp;
150  size_t ntmp = nbuf;
151  nbuf = other.nbuf;
152  other.nbuf = ntmp;
153  }
154 
155  private:
156  T* buf;
157  size_t nbuf;
158 
159  buffer(const buffer<T>&);
160  buffer<T>& operator=(const buffer<T>&);
161  }; // buffer
162 
163 protected:
164  fileManager();
165  ~fileManager();
166 
167  void recordFile(roFile*);
168  void unrecordFile(roFile*);
169 
170  // parameters for controlling the resource usage
174  static uint64_t maxBytes;
176  static unsigned int maxOpenFiles;
177 
178  // not implemented, to prevent compiler from generating these functions
179  fileManager(const fileManager& rhs);
180  fileManager& operator=(const fileManager& rhs);
181 
182 private:
183  typedef std::map< const char*, roFile*,
184  std::less< const char* > > fileList;
185  typedef std::set< const cleaner* > cleanerList;
186  typedef std::set< const char*, std::less< const char* > > nameList;
188  fileList mapped;
190  fileList incore;
192  nameList reading;
194  cleanerList cleaners;
196  double page_count;
198  uint32_t minMapSize;
200  uint32_t nwaiting;
202  pthread_cond_t readCond;
203 
205  //mutable pthread_rwlock_t lock;
207  mutable pthread_mutex_t mutex;
210  mutable pthread_cond_t cond;
211 
213  static time_t hbeat;
215  static uint32_t pagesize;
216 
217  int unload(size_t size); // try to unload size bytes
218  void invokeCleaners() const;// invoke external cleaners
219  //inline void gainWriteAccess(const char* m) const;
220 
221  // class writeLock;
222  // class softWriteLock;
223  // friend class writeLock;
224  // friend class softWriteLock;
225 }; // class fileManager
226 
237 class FASTBIT_CXX_DLLSPEC ibis::fileManager::storage {
238 public:
239  storage();
240  explicit storage(size_t n); // allocate n bytes
241  virtual ~storage() {clear();}
242 
243  storage(const char* fname, const off_t begin, const off_t end);
244  storage(const int fdes, const off_t begin, const off_t end);
245  storage(const char* begin, const char* end);
246  storage(char* addr, size_t num);
247  storage(const storage& rhs);
248  storage& operator=(const storage& rhs);
249  void copy(const storage& rhs);
250 
253  const char* filename() const {return name;}
254 
256  bool empty() const {return (m_begin == 0 || m_begin >= m_end);}
258  size_t size() const {
259  return (m_begin!=0 && m_begin<m_end ? m_end-m_begin : 0);}
261  size_t bytes() const {
262  return (m_begin!=0 && m_begin<m_end ? m_end-m_begin : 0);}
263  void enlarge(size_t nelm=0);
264 
266  virtual void* release();
268  char* begin() {return m_begin;}
270  const char* end() const {return m_end;}
272  const char* begin() const {return m_begin;}
274  char operator[](size_t i) const {return m_begin[i];}
275 
276  virtual void beginUse();
277  virtual void endUse();
279  unsigned inUse() const {return nref();}
281  unsigned pastUse() const {return nacc;}
282 
284  virtual bool isFileMap() const {return false;}
285  // IO functions
286  virtual void printStatus(std::ostream& out) const;
287  off_t read(const char* fname, const off_t begin, const off_t end);
288  off_t read(const int fdes, const off_t begin, const off_t end);
289  void write(const char* file) const;
290 
291  inline void swap(storage& rhs) throw ();
292 // // compares storage objects according to starting addresses
293 // struct less : std::binary_function< storage*, storage*, bool > {
294 // bool operator()(const storage* x, const storage* y) const {
295 // return (x->begin() < y->begin());
296 // }
297 // };
298 
299 protected:
301  char* name;
303  char* m_begin;
305  char* m_end;
307  unsigned nacc;
310 
311  virtual void clear(); // free memory/close file
312 }; // class fileManager::storage
313 
322 class FASTBIT_CXX_DLLSPEC ibis::fileManager::roFile
323  : public ibis::fileManager::storage {
324 public:
325  virtual ~roFile() {clear();}
326 
327  // functions for recording access statistics
328  virtual void beginUse();
329  virtual void endUse();
330  // is the read-only file mapped ?
331  virtual bool isFileMap() const {return (mapped != 0);}
332  int disconnectFile();
333 
334  // IO functions
335  virtual void printStatus(std::ostream& out) const;
336  void read(const char* file);
337 #if defined(HAVE_FILE_MAP)
338  void mapFile(const char* file);
339 #endif
340 
341 // // compares storage objects according to file names
342 // struct less : std::binary_function< roFile*, roFile*, bool > {
343 // bool operator()(const roFile* x, const roFile* y) const {
344 // return strcmp(x->filename(), y->filename()) < 0;
345 // }
346 // };
347 protected:
348  roFile();
349  // Read the whole file into memory.
350  void doRead(const char* file);
351  // Read the specified segment of the file into memory.
352  void doRead(const char* file, off_t b, off_t e);
353 #if defined(HAVE_FILE_MAP)
354  void doMap(const char* file, off_t b, off_t e, int opt=0);
355 #endif
356 
360  float score() const {
361  float sc = FLT_MAX;
362  time_t now = time(0);
363  if (opened >= now) {
364  sc = static_cast<float>(1e-4 * size() + nacc);
365  }
366  else if (lastUse >= now) {
367  sc = static_cast<float>(sqrt(5e-6*size()) + nacc +
368  (now - opened));
369  }
370  else {
371  sc = static_cast<float>((sqrt(1e-6*size() + now - opened) +
372  (static_cast<double>(nacc) /
373  (now - opened))) / (now - lastUse));
374  }
375  return sc;
376  }
377 
378  friend class ibis::fileManager;
379  virtual void clear(); // free memory/close file
380  virtual void* release() {return 0;}
381 
382  void printBody(std::ostream& out) const;
383 
384 private:
386  time_t opened;
388  time_t lastUse;
390  unsigned mapped;
391 
392 #if defined(_WIN32) && defined(_MSC_VER)
393  HANDLE fdescriptor; // HANDLE to the open file
394  HANDLE fmap; // HANDLE to the mapping object
395  LPVOID map_begin; // actual address returned by MapViewOfFile
396 #elif (HAVE_MMAP+0 > 0)
397  int fdescriptor; // descriptor of the open file
398  size_t fsize; // the size of the mapped portion of file
399  void *map_begin; // actual address returned by mmap
400 #endif
401 
402  // not implemented, to prevent automatic generation
403  roFile(const roFile&);
404  roFile& operator=(const roFile&);
405 }; // class fileManager::roFile
406 
407 #if defined(HAVE_FILE_MAP)
408 class FASTBIT_CXX_DLLSPEC ibis::fileManager::rofSegment
412  : public ibis::fileManager::roFile {
413 public:
414  rofSegment(const char *fn, off_t b, off_t e);
415  virtual ~rofSegment() {};
416  virtual void printStatus(std::ostream& out) const;
417 
418 private:
419  rofSegment(); // no default constructor
420  rofSegment(const rofSegment&); // no copy constructor
421  rofSegment& operator=(const rofSegment&); // no assignment operator
422 
423  std::string filename_; // name of the file
424  off_t begin_, end_; // the start and the end address of the file map
425 }; // ibis::fileManager::rofSegment
426 #endif
427 
428 // /// A write lock for controlling access to the two internal lists.
429 // class ibis::fileManager::writeLock {
430 // public:
431 // /// Constructor. Acquires a write lock.
432 // writeLock(const char* m) : mesg(m)
433 // {ibis::fileManager::instance().gainWriteAccess(mesg);}
434 // /// Destructor. Releases the write lock.
435 // ~writeLock() {ibis::fileManager::instance().releaseAccess(mesg);}
436 // private:
437 // const char* mesg;
438 
439 // writeLock(const writeLock&);
440 // writeLock& operator=(const writeLock&);
441 // }; // ibis::fileManager::writeLock
442 
443 // /// A soft write lock for controlling access to the two internal lists.
444 // class ibis::fileManager::softWriteLock {
445 // public:
446 // softWriteLock(const char* m);
447 // ~softWriteLock();
448 // /// Has a write lock been acquired? Returns true or false to indicate
449 // /// yes or no.
450 // bool isLocked() const {return(locked_==0);}
451 
452 // private:
453 // const char* mesg;
454 // const int locked_;
455 
456 // softWriteLock(const softWriteLock&);
457 // softWriteLock& operator=(const softWriteLock&);
458 // }; // ibis::fileManager::softWriteLock
459 
460 inline uint64_t ibis::fileManager::bytesFree() {
461  if (maxBytes == 0)
465 } // ibis::fileManager::bytesFree
466 
467 // /// Release the read/write lock.
468 // inline void ibis::fileManager::releaseAccess(const char* mesg) const {
469 // int ierr = pthread_rwlock_unlock(&lock);
470 // if (0 == ierr) {
471 // LOGGER(ibis::gVerbose > 9)
472 // << "fileManager::releaseAccess on "
473 // << static_cast<const void*>(&lock) << " for " << mesg;
474 // }
475 // else {
476 // LOGGER(ibis::gVerbose >= 0)
477 // << "Warning -- fileManager::releaseAccess on "
478 // << static_cast<const void*>(&lock) << " for " << mesg
479 // << " failed with the error code " << ierr << " -- "
480 // << strerror(ierr);
481 // }
482 // } // ibis::fileManager::releaseAccess
483 
484 // /// Gain read access. It blocks when waiting to acquire the read lock.
485 // inline void ibis::fileManager::gainReadAccess(const char* mesg) const {
486 // int ierr = pthread_rwlock_rdlock(&lock);
487 // if (0 == ierr) {
488 // LOGGER(ibis::gVerbose > 9)
489 // << "fileManager::gainReadAccess on "
490 // << static_cast<const void*>(&lock) << " for " << mesg;
491 // }
492 // else {
493 // LOGGER(ibis::gVerbose >= 0)
494 // << "Warning -- fileManager::gainReadAccess on "
495 // << static_cast<const void*>(&lock) << " for " << mesg
496 // << " failed with the error code " << ierr << " -- "
497 // << strerror(ierr);
498 // }
499 // } // ibis::fileManager::gainReadAccess
500 
501 // /// Gain write access. It blocks when waiting to acquire the write lock.
502 // inline void ibis::fileManager::gainWriteAccess(const char* mesg) const {
503 // int ierr = pthread_rwlock_wrlock(&lock);
504 // if (0 == ierr) {
505 // LOGGER(ibis::gVerbose > 9)
506 // << "fileManager::gainWriteAccess on "
507 // << static_cast<const void*>(&lock) << " for " << mesg;
508 // }
509 // else {
510 // LOGGER(ibis::gVerbose >= 0)
511 // << "Warning -- fileManager::gainWriteAccess on "
512 // << static_cast<const void*>(&lock) << " for " << mesg
513 // << " failed with the error code " << ierr << " -- "
514 // << strerror(ierr);
515 // }
516 // } // ibis::fileManager::gainWriteAccess
517 
521 inline void ibis::fileManager::recordPages(off_t start, off_t stop) {
522  if (stop - start > 0) {
523  start = (start / pagesize) * pagesize;
524  page_count += ceil(static_cast<double>((stop - start)) / pagesize);
525  }
526 } // ibis::fileManager::recordPages
527 
528 inline void
529 ibis::fileManager::increaseUse(size_t inc, const char* evt) {
531  LOGGER(inc > 0 && evt != 0 && *evt != 0 && ibis::gVerbose > 9)
532  << evt << " added " << inc << " bytes to increase totalBytes to "
534 } // ibis::fileManager::increaseUse
535 
536 inline void
537 ibis::fileManager::decreaseUse(size_t dec, const char* evt) {
539  LOGGER(dec > 0 && evt != 0 && *evt != 0 && ibis::gVerbose > 9)
540  << evt << " removed " << dec << " bytes to decrease totalBytes to "
542 } // ibis::fileManager::decreaseUse
543 
553 inline void
555  {char* tmp = name; name = rhs.name; rhs.name = tmp;}
556  {char* tmp = m_begin; m_begin = rhs.m_begin; rhs.m_begin = tmp;}
557  {char* tmp = m_end; m_end = rhs.m_end; rhs.m_end = tmp;}
558  {unsigned itmp = nacc; nacc = rhs.nacc; rhs.nacc = itmp;}
559  //nref.swap(rhs.nref);
560 } // ibis::fileManager::storage::swap
561 #endif // IBIS_FILEMANAGER_H
storage()
Constructor. Allocate no real storage.
Definition: fileManager.cpp:1844
int64_t write(int, const void *, int64_t)
A wrapper over POSIX write function.
Definition: util.cpp:999
void flushFile(const char *name)
Close the file, remove the record about it from the file manager.
Definition: fileManager.cpp:221
virtual void clear()
Freeing the storage allocated.
Definition: fileManager.cpp:2548
~fileManager()
Destructor.
Definition: fileManager.cpp:692
static storage * getFileSegment(const char *name, const int fdes, const off_t b, const off_t e)
Retrieve a portion of a file content.
Definition: fileManager.cpp:1248
A unsigned 64-bit shared integer class.
Definition: util.h:1086
static time_t iBeat()
Returns the value of a simple counter. It is not thread-safe!
Definition: fileManager.h:54
T & operator[](size_t i)
Return the ith value. It does not perform array bounds check!
Definition: fileManager.h:136
const double & pageCount() const
Returns the number of pages accessed by function read from stdlib.h.
Definition: fileManager.h:56
static unsigned int maxOpenFiles
The Maximum number of files that can be kept open at the same time.
Definition: fileManager.h:176
virtual void beginUse()
Record a new active reference to this object.
Definition: fileManager.cpp:2464
size_t size() const
Return the size (bytes) of the object.
Definition: fileManager.h:258
The storage class treats all memory as char*.
Definition: fileManager.h:237
char * begin()
Starting address of the storage object.
Definition: fileManager.h:268
virtual void beginUse()
Start using a file. Increments the active reference.
Definition: fileManager.cpp:2510
const char * end() const
Ending address of the storage object.
Definition: fileManager.h:270
virtual bool isFileMap() const
Is the storage a file map ?
Definition: fileManager.h:284
static ibis::util::sharedInt64 totalBytes
The total number of bytes of all managed objects.
Definition: fileManager.h:172
void swap(buffer< T > &other)
Swap the content of two buffers.
Definition: fileManager.h:146
char * name
Name of the file. NULL (0) if no file is involved.
Definition: fileManager.h:301
The current implementation of FastBit is code named IBIS; most data structures and functions are in t...
Definition: bord.h:16
Definition: util.h:342
const char * begin() const
Starting address of the storage object.
Definition: fileManager.h:272
bool empty() const
Is the storage object empty?
Definition: fileManager.h:256
static int adjustCacheSize(uint64_t)
Change the size of memory cache allocated to the file manager.
Definition: fileManager.cpp:373
virtual void clear()
Actually freeing the storage allocated.
Definition: fileManager.cpp:2226
float score() const
The function assigns a score to a file.
Definition: fileManager.h:360
static uint64_t maxBytes
The maximum number of bytes allowed.
Definition: fileManager.h:174
size_t size() const
The number of elements in the buffer. NOT the number of bytes.
Definition: fileManager.h:142
void removeCleaner(const cleaner *cl)
Unregister the cleaner functor.
Definition: fileManager.cpp:499
virtual void * release()
Release the control of the memory to the caller as a raw pointer.
Definition: fileManager.cpp:2261
virtual void * release()
Release the control of the memory to the caller as a raw pointer.
Definition: fileManager.h:380
void unrecordFile(roFile *)
Remove the file name from the list of files tracked.
Definition: fileManager.cpp:780
char * m_end
End of the storage.
Definition: fileManager.h:305
uint64_t getMaxMmapBytes() const
Return the size in bytes of files that are memory mapped.
Definition: fileManager.cpp:399
static uint32_t pageSize()
Returns the page size (in bytes) used by the file system.
Definition: fileManager.h:58
A buffer is intended to be a temporary workspace in memory.
Definition: fileManager.h:128
unsigned pastUse() const
Number of past accesses to this object.
Definition: fileManager.h:281
void swap(storage &rhs)
Swap the content of the storage objects.
Definition: fileManager.h:554
T * address() const
Address of the buffer allocated.
Definition: fileManager.h:140
storage & operator=(const storage &rhs)
Assignment operator.
Definition: fileManager.cpp:2189
This fileManager is intended to allow different objects to share the same open file.
Definition: fileManager.h:23
int getFile(const char *name, array_t< T > &arr, ACCESS_PREFERENCE pref=MMAP_LARGE_FILES)
Given a file name, place the content in an array_t.
Definition: fileManager.cpp:122
void doRead(const char *file)
Read the content of a file into memory.
Definition: fileManager.cpp:2675
Template array_t is a replacement of std::vector.
Definition: array_t.h:24
ACCESS_PREFERENCE
Hint passed to the function getFile.
Definition: fileManager.h:29
ibis::util::sharedInt32 nref
Number of (active) references to this storage.
Definition: fileManager.h:309
Defines minor utility functions and common classes used by FastBit.
virtual void printStatus(std::ostream &out) const
Print information about the storage object to the specified output stream.
Definition: fileManager.cpp:2610
static uint64_t bytesInUse()
Returns the number of bytes currently on records.
Definition: fileManager.h:115
const T & operator[](size_t i) const
Return the ith value. It does not perform array bounds check!
Definition: fileManager.h:138
unsigned nacc
Number of accesses in the past.
Definition: fileManager.h:307
void flushDir(const char *name)
Close all files in the named directory, including subdirectories.
Definition: fileManager.cpp:268
void printStatus(std::ostream &out) const
Prints status information about the file manager.
Definition: fileManager.cpp:182
char operator[](size_t i) const
Unchecked index operator. Returns the character at position i.
Definition: fileManager.h:274
void copy(const storage &rhs)
Copy function. Make an in-meory copy following the copy-and-swap idiom.
Definition: fileManager.cpp:2197
void enlarge(size_t nelm=0)
Enlarge the current storage object.
Definition: fileManager.cpp:2206
void clear()
Close all files and remove all records of them.
Definition: fileManager.cpp:422
void addCleaner(const cleaner *cl)
Register an external cleaner functor.
Definition: fileManager.cpp:489
static fileManager & instance()
Returns a pointer to the one and only file manager.
Definition: fileManager.cpp:507
virtual void endUse()
Record the termination of an active reference.
Definition: fileManager.cpp:2475
void signalMemoryAvailable() const
Signal to the file manager that some memory have been freed.
Definition: fileManager.cpp:1631
size_t bytes() const
Return the number of bytes contained in the object.
Definition: fileManager.h:261
This class manages content of a whole (read-only) file.
Definition: fileManager.h:322
void recordFile(roFile *)
Record a newly allocated storage in the two lists.
Definition: fileManager.cpp:705
virtual void printStatus(std::ostream &out) const
Print information about the storage object to the specified output stream.
Definition: fileManager.cpp:2295
char * m_begin
Beginning of the storage.
Definition: fileManager.h:303
roFile()
Constructor.
Definition: fileManager.cpp:2496
static uint64_t bytesFree()
Return the number of bytes free.
Definition: fileManager.h:460
unsigned int getMaxOpenMmapFiles() const
Return the count of files that are memory mapped.
Definition: fileManager.cpp:394
const char * filename() const
Pointer to the file name supporting this storage object.
Definition: fileManager.h:253
fileManager()
The protected constructor of the ibis::fileManager class.
Definition: fileManager.cpp:538
void recordPages(off_t start, off_t stop)
Given the starting and ending addresses, this function computes the number of pages involved...
Definition: fileManager.h:521
unsigned inUse() const
Number of current accesses to this object.
Definition: fileManager.h:279
virtual void endUse()
Stop using a file. Decrement the active reference count.
Definition: fileManager.cpp:2526
A shared unsigned 32-bit integer class.
Definition: util.h:945
A function object to be used to register external cleaners.
Definition: fileManager.h:66
int tryGetFile(const char *name, array_t< T > &arr, ACCESS_PREFERENCE pref=MMAP_LARGE_FILES)
Given a file name, place the content in an array_t.
Definition: fileManager.cpp:155
off_t read(const char *fname, const off_t begin, const off_t end)
Read a part of a file.
Definition: fileManager.cpp:2315
virtual bool isFileMap() const
Is the storage a file map ?
Definition: fileManager.h:331
static uint64_t currentCacheSize()
A read lock on the file manager. Any object using a file under the.
Definition: fileManager.h:111
int disconnectFile()
Disconnect the storage object from the file.
Definition: fileManager.cpp:2596

Make It A Bit Faster
Contact us
Disclaimers
FastBit source code
FastBit mailing list archive