casacore
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
MultiFile.h
Go to the documentation of this file.
1 //# MultiFile.h: Class to combine multiple files in a single one
2 //# Copyright (C) 2014
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id: RegularFileIO.h 20551 2009-03-25 00:11:33Z Malte.Marquarding $
27 
28 #ifndef CASA_MULTIFILE_H
29 #define CASA_MULTIFILE_H
30 
31 //# Includes
32 #include <casacore/casa/aips.h>
35 
36 
37 namespace casacore { //# NAMESPACE CASACORE - BEGIN
38 
39  // <summary>
40  // Class to combine multiple files in a single one.
41  // </summary>
42 
43  // <use visibility=export>
44 
45  // <reviewed reviewer="" date="" tests="tMultiFile" demos="">
46  // </reviewed>
47 
48  // <synopsis>
49  // This class is a container file holding multiple virtual files. It is
50  // primarily meant as a container file for the storage manager files of a
51  // table to reduce the number of files used (especially for Lustre) and to
52  // reduce the number of open files (especially when concatenating tables).
53  // <br>A secondary goal is offering the ability to use an IO buffer size
54  // that matches the file system well (large buffer size for e.g. ZFS).
55  // <br>A third goal is offering the ability to use O_DIRECT (if supported by OS)
56  // to tell the OS kernel to bypass its file cache. It makes the I/O behaviour
57  // more predictable which a real-time system might need.
58  //
59  // The SetupNewTable constructor has a StorageOption argument to define
60  // if a MultiFile has to be used and if so, the buffer size to use.
61  // It is also possible to specify that through aipsrc variables.
62  //
63  // A virtual file is spread over multiple (fixed size) data blocks in the
64  // MultiFile. A data block is never shared by multiple files.
65  // For each virtual file MultiFile keeps a MultiFileInfo object telling
66  // the file size and the blocks numbers used for the file. When flushing
67  // the MultiFile, this meta info is written into the header block. If it
68  // does not fit in the header block, the rest is written in a separate "-ext"
69  // file.
70  // if needed, continuation blocks. On open and resync, it is read back.
71  // <br>
72  //
73  // A virtual file is represented by an MFFileIO object, which is derived
74  // from ByteIO and as such part of the casacore IO framework. It makes it
75  // possible for applications to access a virtual file in the same way as
76  // a regular file.
77  //
78  // It is possible to delete a virtual file. Its blocks will be added to
79  // the free block list (which is also stored in the meta info).
80  // </synopsis>
81 
82  // <example>
83  // In principle it is possible to use the MultiFile functions directly.
84  // However, in general it is much easier to use an MFFileIO object
85  // per virtual file as shown below.
86  // <srcblock>
87  // // Create a new MultiFile using a block size of 1 MB.
88  // MultiFile mfile("file.mf', ByteIO::New, 1048576);
89  // // Create a virtual file in it.
90  // MFFileIO mf1(mfile, "mf1", ByteIO::New);
91  // // Use it (for example) as the sink of AipsIO.
92  // AipsIO stream (&mf1);
93  // // Write values.
94  // stream << (Int)10;
95  // stream << True;
96  // // Seek to beginning of file and read data in.
97  // stream.setpos (0);
98  // Int vali;
99  // Bool valb;
100  // stream >> vali >> valb;
101  // </srcblock>
102  // </example>
103 
104  // <todo>
105  // <li> write headers at alternating file positions (for robustness)
106  // <li> possibly write headers entirely at the end if larger than blocksize
107  // </todo>
108 
109 
110  class MultiFile: public MultiFileBase
111  {
112  public:
113  // Open or create a MultiFile with the given name.
114  // Upon creation the block size can be given. If 0, it uses the block size
115  // of the file system the file is on.
116  // If useODirect=True, the O_DIRECT flag in used (if supported). It tells the
117  // kernel to bypass its file cache to have more predictable I/O behaviour.
120 
121  // The destructor flushes and closes the file.
122  virtual ~MultiFile();
123 
124  // Reopen the underlying file for read/write access.
125  // Nothing will be done if the file is writable already.
126  // Otherwise it will be reopened and an exception will be thrown
127  // if it is not possible to reopen it for read/write access.
128  virtual void reopenRW();
129 
130  // Fsync the file (i.e., force the data to be physically written).
131  virtual void fsync();
132 
133  private:
134  // Do the class-specific actions on adding a file.
135  virtual void doAddFile (MultiFileInfo&);
136  // Do the class-specific actions on deleting a file.
137  virtual void doDeleteFile (MultiFileInfo&);
138  // Flush the file itself.
139  virtual void flushFile();
140  // Flush and close the file.
141  virtual void close();
142  // Write the header info.
143  virtual void writeHeader();
144  // Read the header info. If always==False, the info is only read if the
145  // header counter has changed.
146  virtual void readHeader (Bool always=True);
147  // Extend the virtual file to fit lastblk.
148  virtual void extend (MultiFileInfo& info, Int64 lastblk);
149  // Write a data block.
150  virtual void writeBlock (MultiFileInfo& info, Int64 blknr,
151  const void* buffer);
152  // Read a data block.
153  virtual void readBlock (MultiFileInfo& info, Int64 blknr,
154  void* buffer);
155 
156  private:
157  //# Data members
159  int itsFD;
160  };
161 
162 
163 } //# NAMESPACE CASACORE - END
164 
165 #endif
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
int Int
Definition: aipstype.h:50
virtual void writeBlock(MultiFileInfo &info, Int64 blknr, const void *buffer)
Write a data block.
Class to combine multiple files in a single one.
Definition: MultiFile.h:110
Abstract base class to combine multiple files in a single one.
const vector< MultiFileInfo > & info() const
Get the info object (for test purposes mainly).
Helper class for MultiFileBase containing info per internal file.
Definition: MultiFileBase.h:75
virtual void reopenRW()
Reopen the underlying file for read/write access.
virtual void doAddFile(MultiFileInfo &)
Do the class-specific actions on adding a file.
virtual void writeHeader()
Write the header info.
virtual void extend(MultiFileInfo &info, Int64 lastblk)
Extend the virtual file to fit lastblk.
virtual void close()
Flush and close the file.
Bool useODirect() const
Will O_DIRECT be used?
Class for unbuffered IO on a file.
Definition: FiledesIO.h:88
virtual ~MultiFile()
The destructor flushes and closes the file.
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
virtual void readBlock(MultiFileInfo &info, Int64 blknr, void *buffer)
Read a data block.
virtual void doDeleteFile(MultiFileInfo &)
Do the class-specific actions on deleting a file.
const Bool False
Definition: aipstype.h:44
OpenOption
Define the possible ByteIO open options.
Definition: ByteIO.h:65
virtual void readHeader(Bool always=True)
Read the header info.
Int64 blockSize() const
Get the block size used.
virtual void flushFile()
Flush the file itself.
String: the storage and methods of handling collections of characters.
Definition: String.h:225
virtual void fsync()
Fsync the file (i.e., force the data to be physically written).
const Bool True
Definition: aipstype.h:43
MultiFile(const String &name, ByteIO::OpenOption, Int blockSize=0, Bool useODirect=False)
Open or create a MultiFile with the given name.