FasTensor  1.0.0
Transform Supercomputing for AI
ft_endpoint_hdf5.h
Go to the documentation of this file.
1 
2 /*
3 ****************************
4 
5 FasTensor (FT) Copyright (c) 2021, The Regents of the University of
6 California, through Lawrence Berkeley National Laboratory (subject to
7 receipt of any required approvals from the U.S. Dept. of Energy).
8 All rights reserved.
9 
10 If you have questions about your rights to use or distribute this software,
11 please contact Berkeley Lab's Intellectual Property Office at
13 
14 NOTICE. This Software was developed under funding from the U.S. Department
15 of Energy and the U.S. Government consequently retains certain rights. As
16 such, the U.S. Government has been granted for itself and others acting on
17 its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the
18 Software to reproduce, distribute copies to the public, prepare derivative
19 works, and perform publicly and display publicly, and to permit others to do so.
20 
21 
22 ****************************
23 
24 
25 *** License Agreement ***
26 
27 FasTensor (FT) Copyright (c) 2021, The Regents of the University of
28 California, through Lawrence Berkeley National Laboratory (subject to
29 receipt of any required approvals from the U.S. Dept. of Energy).
30 All rights reserved.
31 
32 Redistribution and use in source and binary forms, with or without
33 modification, are permitted provided that the following conditions are met:
34 
35 (1) Redistributions of source code must retain the above copyright notice,
36 this list of conditions and the following disclaimer.
37 
38 (2) Redistributions in binary form must reproduce the above copyright
39 notice, this list of conditions and the following disclaimer in the
40 documentation and/or other materials provided with the distribution.
41 
42 (3) Neither the name of the University of California, Lawrence Berkeley
43 National Laboratory, U.S. Dept. of Energy nor the names of its contributors
44 may be used to endorse or promote products derived from this software
45 without specific prior written permission.
46 
47 
48 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
49 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
52 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
53 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
54 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
55 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
56 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
57 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
58 POSSIBILITY OF SUCH DAMAGE.
59 
60 You are under no obligation whatsoever to provide any bug fixes, patches,
61 or upgrades to the features, functionality or performance of the source
62 code ("Enhancements") to anyone; however, if you choose to make your
63 Enhancements available either publicly, or directly to Lawrence Berkeley
64 National Laboratory, without imposing a separate written license agreement
65 for such Enhancements, then you hereby grant the following license: a
66 non-exclusive, royalty-free perpetual license to install, use, modify,
67 prepare derivative works, incorporate into other computer software,
68 distribute, and sublicense such enhancements or derivative works thereof,
69 in binary and source code form.
70 */
71 
80 #ifndef END_POINT_HDF5_H
81 #define END_POINT_HDF5_H
82 
83 #include "ft_type.h"
84 #include "ft_endpoint.h"
85 #include "ft_xdmf.h"
86 
87 #include <string>
88 #include <iostream>
89 #include <vector>
90 #include <math.h>
91 
92 #include "hdf5.h" //right now, we only have code for HDF5
93 
94 #define HDF5_ENABLE_MPI_IO 0
95 #define HDF5_DISABLE_MPI_IO 1
96 #define HDF5_ENABLE_COLLECTIVE_IO 2
97 #define HDF5_DISABLE_COLLECTIVE_IO 3
98 
99 //For some old code
100 #define OP_ENABLE_MPI_IO HDF5_ENABLE_MPI_IO
101 #define OP_DISABLE_MPI_IO HDF5_DISABLE_MPI_IO
102 #define OP_ENABLE_COLLECTIVE_IO HDF5_ENABLE_COLLECTIVE_IO
103 #define OP_DISABLE_COLLECTIVE_IO HDF5_DISABLE_COLLECTIVE_IO
104 
105 //
106 //I/O layer
107 class EndpointHDF5 : public Endpoint
108 {
109 private:
110  hid_t fid = -1, gid = -1, did = -1;
111  hid_t dataspace_id = -1;
112  std::string fn_str, gn_str, dn_str;
113  hid_t plist_id = H5P_DEFAULT, plist_cio_id = H5P_DEFAULT;
114  hid_t mem_type, disk_type;
115  bool is_mpi_enabled = false;
116 
117 public:
123  EndpointHDF5(std::string endpoint_info_p)
124  {
125  endpoint_info = endpoint_info_p;
127  SetOpenFlag(false);
128  SetRwFlag(H5F_ACC_RDONLY);
130  EnableMPIIO();
132  is_mpi_enabled = true;
133  }
139  {
140  SetOpenFlag(false);
141  SetRwFlag(H5F_ACC_RDONLY);
143  EnableMPIIO();
145  is_mpi_enabled = true;
146  }
147 
153  EndpointHDF5(int no_mpi)
154  {
155  //endpoint_info = endpoint_info_p;
156  //ParseEndpointInfo();
157  SetOpenFlag(false);
158  SetRwFlag(H5F_ACC_RDONLY);
160  //EnableMPIIO();
161  //EnableCollectiveIO();
162  }
163 
165  {
166  Close();
167  if (is_mpi_enabled)
168  {
169  DisableMPIIO();
171  }
172  }
178  int ExtractMeta() override;
184  int PrintInfo() override;
185 
191  int Create() override;
192 
198  int Open() override;
199 
208  int Read(std::vector<unsigned long long> start, std::vector<unsigned long long> end, void *data) override;
209 
218  int Write(std::vector<unsigned long long> start, std::vector<unsigned long long> end, void *data) override;
219 
225  int Close() override;
226 
227  void Map2MyType() override;
228 
229  void Map2MyTypeParameters(FTDataType ft_type, hid_t &mem_type, hid_t &disk_type);
230 
231  void EnableCollectiveIO() override;
232 
233  void DisableCollectiveIO() override;
234 
235  void EnableMPIIO();
236 
237  void DisableMPIIO();
238 
244  int ParseEndpointInfo() override;
245 
252  //int SpecialOperator(int opt_code, std::string parameter) override;
253 
260  int Control(int opt_code, std::vector<std::string> &parameter_v) override;
261 
269  int WriteAttribute(const std::string &name, const void *data, FTDataType data_type_p, const size_t &data_length_p = 0) override;
270 
277  int ReadAllAttributeName(std::vector<std::string> &attr_name);
278 
286  int ReadAttribute(const std::string &name, void *data, FTDataType data_type_p, const size_t &data_length_p = 0) override;
287 
288  int GetAttributeSize(const std::string &name, FTDataType data_type_p) override;
289 
290  int CreateXDMF();
291 };
292 #endif
Definition: ft_endpoint_hdf5.h:108
int ParseEndpointInfo() override
parse endpoint_info to my own info In HDF5, it map endpoint_info to filename, group name and datasetn...
Definition: ft_endpoint_hdf5.cpp:523
int GetAttributeSize(const std::string &name, FTDataType data_type_p) override
Definition: ft_endpoint_hdf5.cpp:762
int ReadAllAttributeName(std::vector< std::string > &attr_name)
Read all attribute name.
Definition: ft_endpoint_hdf5.cpp:613
int Create() override
create the endpoint
Definition: ft_endpoint_hdf5.cpp:97
~EndpointHDF5()
Definition: ft_endpoint_hdf5.h:164
EndpointHDF5()
Construct a new Endpoint in HDF5 Nothing to do there, can be used as sub-endpoint of directory.
Definition: ft_endpoint_hdf5.h:138
int PrintInfo() override
print information about the endpoint
Definition: ft_endpoint_hdf5.cpp:406
int WriteAttribute(const std::string &name, const void *data, FTDataType data_type_p, const size_t &data_length_p=0) override
Set the Attribute object.
Definition: ft_endpoint_hdf5.cpp:651
EndpointHDF5(int no_mpi)
Construct a new EndpointHDF5 object without MPI.
Definition: ft_endpoint_hdf5.h:153
void Map2MyType() override
call the finalize to close everything (like call Destractor)
Definition: ft_endpoint_hdf5.cpp:412
void Map2MyTypeParameters(FTDataType ft_type, hid_t &mem_type, hid_t &disk_type)
Definition: ft_endpoint_hdf5.cpp:467
int ReadAttribute(const std::string &name, void *data, FTDataType data_type_p, const size_t &data_length_p=0) override
Get the Attribute object.
Definition: ft_endpoint_hdf5.cpp:706
void DisableMPIIO()
Definition: ft_endpoint_hdf5.cpp:398
int Write(std::vector< unsigned long long > start, std::vector< unsigned long long > end, void *data) override
write the data to the end-point
Definition: ft_endpoint_hdf5.cpp:272
int Control(int opt_code, std::vector< std::string > &parameter_v) override
call a special operator on endpoint such as, enable collective I/O for HDF5 dump file from MEMORY to ...
Definition: ft_endpoint_hdf5.cpp:584
int Read(std::vector< unsigned long long > start, std::vector< unsigned long long > end, void *data) override
read the data from end-point
Definition: ft_endpoint_hdf5.cpp:225
void DisableCollectiveIO() override
Definition: ft_endpoint_hdf5.cpp:382
int ExtractMeta() override
extracts metadata, possbile endpoint_ranks/endpoint_dim_size/data_element_type
Definition: ft_endpoint_hdf5.cpp:82
int Close() override
close the end-point
Definition: ft_endpoint_hdf5.cpp:342
int CreateXDMF()
Definition: ft_endpoint_hdf5.cpp:823
void EnableMPIIO()
Definition: ft_endpoint_hdf5.cpp:390
int Open() override
open the endpoint
Definition: ft_endpoint_hdf5.cpp:177
void EnableCollectiveIO() override
Definition: ft_endpoint_hdf5.cpp:373
EndpointHDF5(std::string endpoint_info_p)
Construct a new EndpointHDF5 object.
Definition: ft_endpoint_hdf5.h:123
Define the class for the Endpoint used by ArrayUDF to store the data. It contains basic infomation fo...
Definition: ft_endpoint.h:106
void SetEndpointType(AuEndpointType endpoint_type_p)
Set the Endpoint Type object.
Definition: ft_endpoint.cpp:355
void SetOpenFlag(bool open_flag_p)
Definition: ft_endpoint.cpp:156
std::string endpoint_info
Definition: ft_endpoint.h:109
void SetRwFlag(unsigned read_write_flag_p)
Definition: ft_endpoint.cpp:171
@ EP_HDF5
Definition: ft_type.h:96
AuEndpointDataType
Definition: ft_type.h:118