casacore
|
The main class for the Dysco storage manager. More...
#include <dyscostman.h>
Public Member Functions | |
DyscoStMan (unsigned dataBitRate, unsigned weightBitRate, const casacore::String &name="DyscoStMan") | |
Convenience constructor to create a new storage manager with some settings without having to fill a 'spec' Record. More... | |
void | SetGaussianDistribution () |
Initialize the storage manager to use a Gaussian distribution for the quantization. More... | |
void | SetUniformDistribution () |
Initialize the storage manager to use a Uniform distribution for the quantization (i.e., use a linear quantizer). More... | |
void | SetStudentsTDistribution (double nu) |
Initialize the storage manager to use a Student T distribution for the quantization (i.e., use a linear quantizer). More... | |
void | SetTruncatedGaussianDistribution (double truncationSigma) |
Initialize the storage manager to use a Uniform distribution for the quantization (i.e., use a linear quantizer). More... | |
void | SetNormalization (Normalization normalization) |
Set the type of normalization. More... | |
void | SetStaticSeed (bool staticSeed) |
DyscoStMan (const casacore::String &name, const casacore::Record &spec) | |
This constructor is called by Casa when it needs to create a DyscoStMan. More... | |
DyscoStMan (const DyscoStMan &source) | |
Copy constructor that initializes a storage manager with similar specs. More... | |
~DyscoStMan () | |
Destructor. More... | |
DyscoStMan & | operator= (const DyscoStMan &source)=delete |
Assignment – new dyscostman takes the settings of the source (but not the columns and/or data). More... | |
virtual casacore::DataManager * | clone () const finaloverride |
Polymorphical copy constructor, equal to DyscoStMan(const DyscoStMan&). More... | |
virtual casacore::String | dataManagerType () const finaloverride |
Type of manager. More... | |
virtual casacore::String | dataManagerName () const finaloverride |
Returns the name of this manager as specified during construction. More... | |
virtual casacore::Record | dataManagerSpec () const finaloverride |
Get manager specifications. More... | |
uint | getNRow () const |
Get the number of rows in the measurement set. More... | |
virtual casacore::Bool | canAddRow () const finaloverride |
Whether rows can be added. More... | |
virtual casacore::Bool | canRemoveRow () const finaloverride |
Whether rows can be removed. More... | |
virtual casacore::Bool | canAddColumn () const finaloverride |
Whether columns can be added. More... | |
virtual casacore::Bool | canRemoveColumn () const finaloverride |
Whether columns can be removed. More... | |
Public Member Functions inherited from casacore::DataManager | |
DataManager () | |
Default constructor. More... | |
virtual | ~DataManager () |
void | dataManagerInfo (Record &info) const |
Add SEQNR and SPEC (the DataManagerSpec subrecord) to the info. More... | |
virtual Record | getProperties () const |
Get data manager properties that can be modified. More... | |
virtual void | setProperties (const Record &spec) |
Modify data manager properties given in record fields. More... | |
virtual Bool | isStorageManager () const |
Is the data manager a storage manager? The default is yes. More... | |
virtual Bool | canReallocateColumns () const |
Tell if the data manager wants to reallocate the data manager column objects. More... | |
virtual DataManagerColumn * | reallocateColumn (DataManagerColumn *column) |
Reallocate the column object if it is part of this data manager. More... | |
uInt | sequenceNr () const |
Get the (unique) sequence nr of this data manager. More... | |
uInt | ncolumn () const |
Get the nr of columns in this data manager (can be zero). More... | |
Bool | asBigEndian () const |
Have the data to be stored in big or little endian canonical format? More... | |
const TSMOption & | tsmOption () const |
Get the TSM option. More... | |
MultiFileBase * | multiFile () |
Get the MultiFile pointer (can be 0). More... | |
String | keywordName (const String &keyword) const |
Compose a keyword name from the given keyword appended with the sequence number (e.g. More... | |
String | fileName () const |
Compose a unique filename from the table name and sequence number. More... | |
ByteIO::OpenOption | fileOption () const |
Get the AipsIO option of the underlying file. More... | |
virtual Bool | isRegular () const |
Is this a regular storage manager? It is regular if it allows addition of rows and writing data in them. More... | |
Table & | table () const |
Get the table this object is associated with. More... | |
virtual Bool | canRenameColumn () const |
Does the data manager allow to rename columns? (default yes) More... | |
virtual void | setMaximumCacheSize (uInt nMiB) |
Set the maximum cache size (in bytes) to be used by a storage manager. More... | |
virtual void | showCacheStatistics (std::ostream &) const |
Show the data manager's IO statistics. More... | |
DataManagerColumn * | createScalarColumn (const String &columnName, int dataType, const String &dataTypeId) |
Create a column in the data manager on behalf of a table column. More... | |
DataManagerColumn * | createDirArrColumn (const String &columnName, int dataType, const String &dataTypeId) |
Create a direct array column. More... | |
DataManagerColumn * | createIndArrColumn (const String &columnName, int dataType, const String &dataTypeId) |
Create an indirect array column. More... | |
DataManager * | getClone () const |
Has the object already been cloned? More... | |
void | setClone (DataManager *clone) const |
Set the pointer to the clone. More... | |
Static Public Member Functions | |
static casacore::DataManager * | makeObject (const casacore::String &name, const casacore::Record &spec) |
Create an object with given name and spec. More... | |
static void | registerClass () |
This function makes the DyscoStMan known to casacore. More... | |
Static Public Member Functions inherited from casacore::DataManager | |
static void | registerCtor (const String &type, DataManagerCtor func) |
Register a mapping of a data manager type to its static construction function. More... | |
static DataManagerCtor | getCtor (const String &dataManagerType) |
Get the "constructor" of a data manager (thread-safe). More... | |
static Bool | isRegistered (const String &dataManagerType) |
Test if a data manager is registered (thread-safe). More... | |
static DataManager * | unknownDataManager (const String &dataManagerType, const Record &spec) |
Serve as default function for theirRegisterMap, which catches all unknown data manager types. More... | |
Protected Member Functions | |
uint64_t | nBlocksInFile () const |
The number of rows that are actually stored in the file. More... | |
size_t | nRowsInBlock () const |
Number of rows in one "time-block", i.e. More... | |
size_t | nAntennae () const |
Number of antennae used in a time block. More... | |
size_t | getBlockIndex (uint64_t row) const |
Return index of block that contains the given measurement set row. More... | |
size_t | getRowWithinBlock (uint64_t row) const |
Return the offset of the row within the block. More... | |
uint64_t | getRowIndex (size_t block) const |
Calculate first measurement set row index of a given block index. More... | |
bool | areOffsetsInitialized () const |
This method returns true when the number of rows per block and the number of antennae per block are known. More... | |
void | initializeRowsPerBlock (size_t rowsPerBlock, size_t antennaCount, bool writeToHeader) |
To be called by a column once it determines rowsPerBlock and antennaCount. More... | |
Protected Member Functions inherited from casacore::DataManager | |
void | decrementNcolumn () |
Decrement number of columns (in case a column is deleted). More... | |
void | setEndian (Bool bigEndian) |
Tell the data manager if big or little endian format is needed. More... | |
void | setTsmOption (const TSMOption &tsmOption) |
Tell the data manager which TSM option to use. More... | |
void | setMultiFile (MultiFileBase *mfile) |
Tell the data manager that MultiFile can be used. More... | |
virtual Bool | hasMultiFileSupport () const |
Does the data manager support use of MultiFile? A derived class has to return True if it can use the MultiFile. More... | |
void | throwDataTypeOther (const String &columnName, int dataType) const |
Throw an exception in case data type is TpOther, because the storage managers (and maybe other data managers) do not support such columns. More... | |
Private Member Functions | |
void | readCompressedData (size_t blockIndex, const DyscoStManColumn *column, unsigned char *dest, size_t size) |
void | writeCompressedData (size_t blockIndex, const DyscoStManColumn *column, const unsigned char *data, size_t size) |
void | readHeader () |
void | writeHeader () |
void | makeEmpty () |
void | setFromSpec (const casacore::Record &spec) |
size_t | getFileOffset (size_t blockIndex) const |
virtual casacore::Bool | flush (casacore::AipsIO &, casacore::Bool doFsync) finaloverride |
Flush and optionally fsync the data. More... | |
virtual void | create64 (casacore::rownr_t nRow) finaloverride |
Let the storage manager create files as needed for a new table. More... | |
virtual casacore::rownr_t | open64 (casacore::rownr_t nRow, casacore::AipsIO &) finaloverride |
Open the storage manager file for an existing table. More... | |
virtual casacore::DataManagerColumn * | makeScalarColumn (const casacore::String &name, int dataType, const casacore::String &dataTypeID) finaloverride |
Create a column in the storage manager on behalf of a table column. More... | |
virtual casacore::DataManagerColumn * | makeDirArrColumn (const casacore::String &name, int dataType, const casacore::String &dataTypeID) finaloverride |
Create a direct array column. More... | |
virtual casacore::DataManagerColumn * | makeIndArrColumn (const casacore::String &name, int dataType, const casacore::String &dataTypeID) finaloverride |
Create an indirect array column. More... | |
virtual casacore::rownr_t | resync64 (casacore::rownr_t nRow) finaloverride |
virtual void | deleteManager () finaloverride |
The data manager will be deleted (because all its columns are requested to be deleted). More... | |
virtual void | prepare () finaloverride |
Prepare the columns, let the data manager initialize itself further. More... | |
virtual void | reopenRW () finaloverride |
Reopen the storage manager files for read/write. More... | |
virtual void | addRow64 (casacore::rownr_t nrrow) finaloverride |
Add rows to the storage manager. More... | |
virtual void | removeRow64 (casacore::rownr_t rowNr) finaloverride |
Delete a row from all columns. More... | |
virtual void | addColumn (casacore::DataManagerColumn *) finaloverride |
Do the final addition of a column. More... | |
virtual void | removeColumn (casacore::DataManagerColumn *) finaloverride |
Remove a column from the data file. More... | |
Private Attributes | |
uint64_t | _nRow |
uint64_t | _nBlocksInFile |
uint32_t | _rowsPerBlock |
uint32_t | _antennaCount |
uint32_t | _blockSize |
unsigned | _headerSize |
std::mutex | _mutex |
std::unique_ptr< std::fstream > | _fStream |
std::string | _name |
unsigned | _dataBitCount |
unsigned | _weightBitCount |
DyscoDistribution | _distribution |
Normalization | _normalization |
double | _studentTNu |
double | _distributionTruncation |
bool | _staticSeed |
std::vector< std::unique_ptr < DyscoStManColumn > > | _columns |
Static Private Attributes | |
static const unsigned short | VERSION_MAJOR |
static const unsigned short | VERSION_MINOR |
Friends | |
class | DyscoStManColumn |
Additional Inherited Members | |
Static Public Attributes inherited from casacore::DataManager | |
static rownr_t | MAXROWNR32 |
Define the highest row number that can be represented as signed 32-bit. More... | |
The main class for the Dysco storage manager.
Definition at line 46 of file dyscostman.h.
dyscostman::DyscoStMan::DyscoStMan | ( | unsigned | dataBitRate, |
unsigned | weightBitRate, | ||
const casacore::String & | name = "DyscoStMan" |
||
) |
Convenience constructor to create a new storage manager with some settings without having to fill a 'spec' Record.
The storage manager will be initialized to AF normalization with a truncated Gaussian distribution for the quantization, and a truncation of sigma = 2.5. To change the settings, use one of the Set...Distribution() methods and SetNormalization().
dataBitRate | The number of bits per float used for visibilities. |
weightBitRate | The number of bits per float used for the weight column. |
name | Storage manager name. |
Referenced by clone(), and makeObject().
dyscostman::DyscoStMan::DyscoStMan | ( | const casacore::String & | name, |
const casacore::Record & | spec | ||
) |
This constructor is called by Casa when it needs to create a DyscoStMan.
Casa will call makeObject() that will call this constructor. When it loads an DyscoStMan for an existing MS, the "spec" parameter will be empty, thus the class should initialize its properties by reading them from the file. The spec
is used to make a new storage manager with specs similar to another one.
name | Name of this storage manager. |
spec | Specs to initialize this class with. |
dyscostman::DyscoStMan::DyscoStMan | ( | const DyscoStMan & | source | ) |
Copy constructor that initializes a storage manager with similar specs.
The columns are not copied: the new manager will be empty.
dyscostman::DyscoStMan::~DyscoStMan | ( | ) |
Destructor.
|
finaloverrideprivatevirtual |
Do the final addition of a column.
Reimplemented from casacore::DataManager.
|
finaloverrideprivatevirtual |
Add rows to the storage manager.
|
inlineprotected |
This method returns true
when the number of rows per block and the number of antennae per block are known.
This is only the case once the first time- block was written to the file.
Definition at line 296 of file dyscostman.h.
References _rowsPerBlock.
Referenced by dyscostman::DyscoStManColumn::areOffsetsInitialized().
|
inlinefinaloverridevirtual |
Whether columns can be added.
true
(but restrictions apply; columns can only be added as long as no writes have been performed on the set). Reimplemented from casacore::DataManager.
Definition at line 202 of file dyscostman.h.
|
inlinefinaloverridevirtual |
Whether rows can be added.
true
Reimplemented from casacore::DataManager.
Definition at line 189 of file dyscostman.h.
|
inlinefinaloverridevirtual |
Whether columns can be removed.
true
(but restrictions apply – still to be checked) Reimplemented from casacore::DataManager.
Definition at line 209 of file dyscostman.h.
|
inlinefinaloverridevirtual |
Whether rows can be removed.
true
(but only rows at the end can actually be removed) Reimplemented from casacore::DataManager.
Definition at line 195 of file dyscostman.h.
|
inlinefinaloverridevirtual |
Polymorphical copy constructor, equal to DyscoStMan(const DyscoStMan&).
Implements casacore::DataManager.
Definition at line 158 of file dyscostman.h.
References DyscoStMan().
|
finaloverrideprivatevirtual |
Let the storage manager create files as needed for a new table.
This allows a column with an indirect array to create its file.
|
inlinefinaloverridevirtual |
Returns the name of this manager as specified during construction.
Reimplemented from casacore::DataManager.
Definition at line 169 of file dyscostman.h.
References _name.
|
finaloverridevirtual |
Get manager specifications.
Includes method settings, etc. Can be used to make a second storage manager with
Reimplemented from casacore::DataManager.
|
inlinefinaloverridevirtual |
Type of manager.
Implements casacore::DataManager.
Definition at line 164 of file dyscostman.h.
|
finaloverrideprivatevirtual |
The data manager will be deleted (because all its columns are requested to be deleted).
So clean up the things needed (e.g. delete files).
Implements casacore::DataManager.
|
finaloverrideprivatevirtual |
Flush and optionally fsync the data.
The AipsIO stream represents the main table file and can be used by virtual column engines to store SMALL amounts of data.
Implements casacore::DataManager.
|
inlineprotected |
Return index of block that contains the given measurement set row.
This can only be calculated after a first time block was written (see areOffsetsInitialized()).
row | A measurement set row. |
Definition at line 269 of file dyscostman.h.
References _rowsPerBlock.
Referenced by dyscostman::DyscoStManColumn::getBlockIndex().
|
inlineprivate |
Definition at line 326 of file dyscostman.h.
References _blockSize, and _headerSize.
|
inline |
Get the number of rows in the measurement set.
Definition at line 183 of file dyscostman.h.
References _nRow.
|
inlineprotected |
Calculate first measurement set row index of a given block index.
block | A block index |
Definition at line 286 of file dyscostman.h.
References _rowsPerBlock.
Referenced by dyscostman::DyscoStManColumn::getRowIndex().
|
inlineprotected |
Return the offset of the row within the block.
This can only be calculated after a first time block was written (see areOffsetsInitialized()).
row | A measurement set row. |
Definition at line 279 of file dyscostman.h.
References _rowsPerBlock.
Referenced by dyscostman::DyscoStManColumn::getRowWithinBlock().
|
protected |
To be called by a column once it determines rowsPerBlock and antennaCount.
rowsPerBlock | Number of measurement set rows in one time block. |
antennaCount | Highest antenna index+1 used in a time block. |
writeToHeader | Write the header? |
Referenced by dyscostman::DyscoStManColumn::initializeRowsPerBlock().
|
finaloverrideprivatevirtual |
Create a direct array column.
Implements casacore::DataManager.
|
private |
|
finaloverrideprivatevirtual |
Create an indirect array column.
Implements casacore::DataManager.
|
inlinestatic |
Create an object with given name and spec.
This methods gets registered in the DataManager "constructor" map. The caller has to delete the object. New class will be initialized via DyscoStMan(const casacore::String& name, const casacore::Record& spec).
Definition at line 219 of file dyscostman.h.
References DyscoStMan().
|
finaloverrideprivatevirtual |
Create a column in the storage manager on behalf of a table column.
The caller will NOT delete the newly created object. Create a scalar column.
Implements casacore::DataManager.
|
inlineprotected |
Number of antennae used in a time block.
This does not have to be equal to the number of antennae stored in the measurement set. This value is only available after a first time block was written (see areOffsetsInitialized()).
Definition at line 260 of file dyscostman.h.
References _antennaCount.
Referenced by dyscostman::DyscoStManColumn::nAntennae().
|
inlineprotected |
The number of rows that are actually stored in the file.
This method is synchronized (i.e., thread-safe).
Definition at line 239 of file dyscostman.h.
References _mutex, and _nBlocksInFile.
Referenced by dyscostman::DyscoStManColumn::nBlocksInFile().
|
inlineprotected |
Number of rows in one "time-block", i.e.
a sequence of rows that belong to the same timestep, spw and field. This value is only available after a first time block was written (see areOffsetsInitialized()).
Definition at line 251 of file dyscostman.h.
References _rowsPerBlock.
Referenced by dyscostman::DyscoStManColumn::nRowsInBlock().
|
finaloverrideprivatevirtual |
Open the storage manager file for an existing table.
Return the number of rows in the data file.
|
delete |
Assignment – new dyscostman takes the settings of the source (but not the columns and/or data).
source | Source manager. |
|
finaloverrideprivatevirtual |
Prepare the columns, let the data manager initialize itself further.
Prepare is called after create/open has been called for all columns. In this way one can be sure that referenced columns are read back and partly initialized.
Reimplemented from casacore::DataManager.
|
private |
Referenced by dyscostman::DyscoStManColumn::readCompressedData().
|
private |
|
static |
This function makes the DyscoStMan known to casacore.
The function is necessary for loading the storage manager from a shared library. It should have this specific name ("register_" + storage manager's name in lowercase) to be able to be automatically called when the library is loaded. That function will forward the call here.
|
finaloverrideprivatevirtual |
Remove a column from the data file.
Reimplemented from casacore::DataManager.
|
finaloverrideprivatevirtual |
Delete a row from all columns.
|
finaloverrideprivatevirtual |
Reopen the storage manager files for read/write.
Reimplemented from casacore::DataManager.
|
finaloverrideprivatevirtual |
|
private |
|
inline |
Initialize the storage manager to use a Gaussian distribution for the quantization.
This method should only be called directly after creating DyscoStMan, before adding columns, and reading/writing data.
In tests with MWA and LOFAR data, the Gaussian distribution showed lesser compression accuracy compared to the truncated Gaussian and uniform distributions.
Definition at line 72 of file dyscostman.h.
References _distribution, and dyscostman::GaussianDistribution.
|
inline |
Set the type of normalization.
This method should only be called directly after creating DyscoStMan, before adding columns, and reading/writing data.
Definition at line 121 of file dyscostman.h.
References _normalization.
|
inline |
Definition at line 125 of file dyscostman.h.
References _staticSeed.
|
inline |
Initialize the storage manager to use a Student T distribution for the quantization (i.e., use a linear quantizer).
This method should only be called directly after creating DyscoStMan, before adding columns, and reading/writing data.
The Student T distribution performed not very well on test sets, and was mainly added for testing.
Definition at line 95 of file dyscostman.h.
References _distribution, _studentTNu, and dyscostman::StudentsTDistribution.
|
inline |
Initialize the storage manager to use a Uniform distribution for the quantization (i.e., use a linear quantizer).
This method should only be called directly after creating DyscoStMan, before adding columns, and reading/writing data.
In tests with MWA and LOFAR data, the truncated Gaussian distribution with a sigma of 1.5 to 2.5 is the recommended distribution.
truncationSigma | At which point the distribution is truncated. Good values are 1.5 to 2.5. |
Definition at line 111 of file dyscostman.h.
References _distribution, _distributionTruncation, and dyscostman::TruncatedGaussianDistribution.
|
inline |
Initialize the storage manager to use a Uniform distribution for the quantization (i.e., use a linear quantizer).
This method should only be called directly after creating DyscoStMan, before adding columns, and reading/writing data.
In tests with MWA and LOFAR data, the Uniform distribution showed very good results, only the truncated Gaussian distribution showed better results for some cases.
Definition at line 84 of file dyscostman.h.
References _distribution, and dyscostman::UniformDistribution.
|
private |
Referenced by dyscostman::DyscoStManColumn::writeCompressedData().
|
private |
|
friend |
Definition at line 308 of file dyscostman.h.
|
private |
Definition at line 389 of file dyscostman.h.
Referenced by nAntennae().
|
private |
Definition at line 390 of file dyscostman.h.
Referenced by getFileOffset().
|
private |
Definition at line 404 of file dyscostman.h.
|
private |
Definition at line 397 of file dyscostman.h.
|
private |
Definition at line 399 of file dyscostman.h.
Referenced by SetGaussianDistribution(), SetStudentsTDistribution(), SetTruncatedGaussianDistribution(), and SetUniformDistribution().
|
private |
Definition at line 401 of file dyscostman.h.
Referenced by SetTruncatedGaussianDistribution().
|
private |
Definition at line 394 of file dyscostman.h.
|
private |
Definition at line 392 of file dyscostman.h.
Referenced by getFileOffset().
|
mutableprivate |
Definition at line 393 of file dyscostman.h.
Referenced by nBlocksInFile().
|
private |
Definition at line 396 of file dyscostman.h.
Referenced by dataManagerName().
|
private |
Definition at line 387 of file dyscostman.h.
Referenced by nBlocksInFile().
|
private |
Definition at line 400 of file dyscostman.h.
Referenced by SetNormalization().
|
private |
Definition at line 386 of file dyscostman.h.
Referenced by getNRow().
|
private |
Definition at line 388 of file dyscostman.h.
Referenced by areOffsetsInitialized(), getBlockIndex(), getRowIndex(), getRowWithinBlock(), and nRowsInBlock().
|
private |
Definition at line 402 of file dyscostman.h.
Referenced by SetStaticSeed().
|
private |
Definition at line 401 of file dyscostman.h.
Referenced by SetStudentsTDistribution().
|
private |
Definition at line 398 of file dyscostman.h.
|
staticprivate |
Definition at line 310 of file dyscostman.h.
|
staticprivate |
Definition at line 310 of file dyscostman.h.