casacore
|
Tiled Data Storage Manager using the shape as id. More...
#include <TiledShapeStMan.h>
Public Member Functions | |
TiledShapeStMan (const String &hypercolumnName, const IPosition &defaultTileShape, uInt64 maximumCacheSize=0) | |
Create a TiledShapeStMan storage manager for the hypercolumn with the given name. More... | |
TiledShapeStMan (const String &hypercolumnName, const Record &spec) | |
~TiledShapeStMan () | |
virtual DataManager * | clone () const |
Clone this object. More... | |
virtual String | dataManagerType () const |
Get the type name of the data manager (i.e. More... | |
virtual Record | dataManagerSpec () const |
Return a record containing data manager specifications and info. More... | |
virtual Bool | canAccessColumn () const |
TiledShapeStMan can access a column if there are 2 hypercubes and the first one is empty. More... | |
virtual TSMCube * | singleHypercube () |
Test if only one hypercube is used by this storage manager. More... | |
virtual void | setShape (rownr_t rownr, TSMCube *hypercube, const IPosition &shape, const IPosition &tileShape) |
Set the shape and tile shape of the given hypercube. More... | |
Public Member Functions inherited from casacore::TiledStMan | |
TiledStMan () | |
Create a TiledStMan. More... | |
TiledStMan (const String &hypercolumnName, uInt maximumCacheSizeMiB) | |
Create a TiledStMan storage manager. More... | |
virtual | ~TiledStMan () |
virtual String | dataManagerName () const |
Get the name given to the storage manager. More... | |
void | setDataManagerName (const String &newHypercolumnName) |
virtual Record | getProperties () const |
Get data manager properties that can be modified. More... | |
virtual void | setProperties (const Record &spec) |
Modify data manager properties. More... | |
void | setDataChanged () |
Set the flag to "data has changed since last flush". More... | |
virtual void | setMaximumCacheSize (uInt nMiB) |
Set the maximum cache size (in MiB) in a non-persistent way. More... | |
uInt | maximumCacheSize () const |
Get the current maximum cache size (in MiB (MibiByte)). More... | |
uInt | cacheSize (rownr_t rownr) const |
Get the current cache size (in buckets) for the hypercube in the given row. More... | |
const IPosition & | hypercubeShape (rownr_t rownr) const |
Get the hypercube shape of the data in the given row. More... | |
const IPosition & | tileShape (rownr_t rownr) const |
Get the tile shape of the data in the given row. More... | |
uInt64 | bucketSize (rownr_t rownr) const |
Get the bucket size (in bytes) of the hypercube in the given row. More... | |
virtual Bool | canChangeShape () const |
Can the tiled storage manager handle changing array shapes? The default is no (but TiledCellStMan can). More... | |
virtual Bool | hasMultiFileSupport () const |
The data manager supports use of MultiFile. More... | |
uInt | calcCacheSize (rownr_t rownr, const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath) const |
Calculate the cache size (in buckets) for accessing the hypercube containing the given row. More... | |
void | setCacheSize (rownr_t rownr, const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, Bool forceSmaller) |
Set the cache size using the calcCacheSize function mentioned above. More... | |
void | setCacheSize (rownr_t rownr, uInt nbuckets, Bool forceSmaller) |
Set the cache size for accessing the hypercube containing the given row. More... | |
void | setHypercubeCacheSize (uInt hypercube, uInt nbuckets, Bool forceSmaller) |
Sets the cache size using the hypercube instead of the row number. More... | |
Bool | userSetCache (rownr_t rownr) const |
Determine if the user set the cache size (using setCacheSize). More... | |
void | emptyCaches () |
Empty the caches used by the hypercubes in this storage manager. More... | |
void | showCacheStatistics (ostream &os) const |
Show the statistics of all caches used. More... | |
uInt64 | getLengthOffset (uInt64 nrPixels, Block< uInt > &dataOffset, Block< uInt > &localOffset, uInt &localTileLength) const |
Get the length of the data for the given number of pixels. More... | |
uInt | nrCoordVector () const |
Get the number of coordinate vectors. More... | |
rownr_t | nrow () const |
Get the nr of rows in this storage manager. More... | |
Bool | canAddRow () const |
Does the storage manager allow to add rows? (yes) More... | |
uInt | nhypercubes () const |
Return the number of hypercubes. More... | |
const TSMCube * | getTSMCube (uInt hypercube) const |
Get the given hypercube. More... | |
TSMCube * | getTSMCube (uInt hypercube) |
const TSMCube * | getHypercube (rownr_t rownr) const |
Get the hypercube in which the given row is stored. More... | |
TSMCube * | makeTSMCube (TSMFile *file, const IPosition &cubeShape, const IPosition &tileShape, const Record &values, Int64 fileOffset=-1) |
Make the correct TSMCube type (depending on tsmOption()). More... | |
void | readTile (char *local, const Block< uInt > &localOffset, const char *external, const Block< uInt > &externalOffset, uInt nrpixels) |
Read a tile and convert the data to local format. More... | |
void | writeTile (char *external, const Block< uInt > &externalOffset, const char *local, const Block< uInt > &localOffset, uInt nrpixels) |
Write a tile after converting the data to external format. More... | |
TSMFile * | getFile (uInt sequenceNumber) |
Get the TSMFile object with the given sequence number. More... | |
virtual rownr_t | open64 (rownr_t nrrow, AipsIO &) |
Open the storage manager for an existing table. More... | |
virtual rownr_t | resync64 (rownr_t nrrow) |
Resync the storage manager with the new file contents. More... | |
virtual void | reopenRW () |
Reopen all files used in this storage manager for read/write access. More... | |
virtual void | deleteManager () |
The data manager will be deleted (because all its columns are requested to be deleted). More... | |
DataManagerColumn * | makeScalarColumn (const String &name, int dataType, const String &dataTypeID) |
Create a column in the storage manager on behalf of a table column. More... | |
DataManagerColumn * | makeDirArrColumn (const String &name, int dataType, const String &dataTypeID) |
Create a direct array column. More... | |
DataManagerColumn * | makeIndArrColumn (const String &name, int dataType, const String &dataTypeID) |
Create an indirect array column. More... | |
Bool | canReallocateColumns () const |
The TiledStMan wants to do reallocateColumn. More... | |
DataManagerColumn * | reallocateColumn (DataManagerColumn *column) |
Reallocate the column object if it is part of this data manager. More... | |
void | checkCubeShape (const TSMCube *hypercube, const IPosition &cubeShape) const |
Check the shape to be set for a hypercube. More... | |
int | coordinateDataType (const String &columnName) const |
Get the data type of the coordinate column with the given name. More... | |
void | initCoordinates (TSMCube *hypercube) |
Initialize the new coordinates for the given cube. More... | |
const TSMDataColumn * | getDataColumn (uInt colnr) const |
Get pointer to data column object. More... | |
Public Member Functions inherited from casacore::DataManager | |
DataManager () | |
Default constructor. More... | |
virtual | ~DataManager () |
void | dataManagerInfo (Record &info) const |
Add SEQNR and SPEC (the DataManagerSpec subrecord) to the info. More... | |
virtual Bool | isStorageManager () const |
Is the data manager a storage manager? The default is yes. More... | |
uInt | sequenceNr () const |
Get the (unique) sequence nr of this data manager. More... | |
uInt | ncolumn () const |
Get the nr of columns in this data manager (can be zero). More... | |
Bool | asBigEndian () const |
Have the data to be stored in big or little endian canonical format? More... | |
const TSMOption & | tsmOption () const |
Get the TSM option. More... | |
MultiFileBase * | multiFile () |
Get the MultiFile pointer (can be 0). More... | |
String | keywordName (const String &keyword) const |
Compose a keyword name from the given keyword appended with the sequence number (e.g. More... | |
String | fileName () const |
Compose a unique filename from the table name and sequence number. More... | |
ByteIO::OpenOption | fileOption () const |
Get the AipsIO option of the underlying file. More... | |
virtual Bool | isRegular () const |
Is this a regular storage manager? It is regular if it allows addition of rows and writing data in them. More... | |
Table & | table () const |
Get the table this object is associated with. More... | |
virtual Bool | canRemoveRow () const |
Does the data manager allow to delete rows? (default no) More... | |
virtual Bool | canAddColumn () const |
Does the data manager allow to add columns? (default no) More... | |
virtual Bool | canRemoveColumn () const |
Does the data manager allow to delete columns? (default no) More... | |
virtual Bool | canRenameColumn () const |
Does the data manager allow to rename columns? (default yes) More... | |
virtual void | showCacheStatistics (std::ostream &) const |
Show the data manager's IO statistics. More... | |
DataManagerColumn * | createScalarColumn (const String &columnName, int dataType, const String &dataTypeId) |
Create a column in the data manager on behalf of a table column. More... | |
DataManagerColumn * | createDirArrColumn (const String &columnName, int dataType, const String &dataTypeId) |
Create a direct array column. More... | |
DataManagerColumn * | createIndArrColumn (const String &columnName, int dataType, const String &dataTypeId) |
Create an indirect array column. More... | |
DataManager * | getClone () const |
Has the object already been cloned? More... | |
void | setClone (DataManager *clone) const |
Set the pointer to the clone. More... | |
Static Public Member Functions | |
static DataManager * | makeObject (const String &dataManagerType, const Record &spec) |
Make the object from the type name string. More... | |
Static Public Member Functions inherited from casacore::TiledStMan | |
static IPosition | makeTileShape (const IPosition &hypercubeShape, Double tolerance=0.5, uInt64 maxNrPixelsPerTile=4 *1024 *1024) |
Derive the tile shape from the hypercube shape for the given number of pixels per tile. More... | |
static IPosition | makeTileShape (const IPosition &hypercubeShape, const Vector< double > &weight, const Vector< double > &tolerance, uInt64 maxNrPixelsPerTile=4 *1024 *1024) |
Static Public Member Functions inherited from casacore::DataManager | |
static void | registerCtor (const String &type, DataManagerCtor func) |
Register a mapping of a data manager type to its static construction function. More... | |
static DataManagerCtor | getCtor (const String &dataManagerType) |
Get the "constructor" of a data manager (thread-safe). More... | |
static Bool | isRegistered (const String &dataManagerType) |
Test if a data manager is registered (thread-safe). More... | |
static DataManager * | unknownDataManager (const String &dataManagerType, const Record &spec) |
Serve as default function for theirRegisterMap, which catches all unknown data manager types. More... | |
Private Member Functions | |
TiledShapeStMan () | |
Create a TiledShapeStMan. More... | |
TiledShapeStMan (const TiledShapeStMan &) | |
Forbid copy constructor. More... | |
TiledShapeStMan & | operator= (const TiledShapeStMan &) |
Forbid assignment. More... | |
virtual IPosition | defaultTileShape () const |
Get the default tile shape. More... | |
void | addRow64 (rownr_t nrrow) |
Add rows to the storage manager. More... | |
Int | findHypercube (const IPosition &shape) |
Find the hypercube for the given shape. More... | |
void | addHypercube (rownr_t rownr, const IPosition &cubeShape, const IPosition &tileShape) |
Add a hypercube. More... | |
void | extendHypercube (rownr_t rownr, uInt cubeNr) |
Extend the hypercube with the given number of elements in the last dimension. More... | |
virtual TSMCube * | getHypercube (rownr_t rownr) |
Get the hypercube in which the given row is stored. More... | |
virtual TSMCube * | getHypercube (rownr_t rownr, IPosition &position) |
Get the hypercube in which the given row is stored. More... | |
virtual void | setupCheck (const TableDesc &tableDesc, const Vector< String > &dataNames) const |
Check if the hypercolumn definition fits this storage manager. More... | |
virtual Bool | flush (AipsIO &, Bool fsync) |
Flush and optionally fsync the data. More... | |
virtual void | create64 (rownr_t nrrow) |
Let the storage manager create files as needed for a new table. More... | |
virtual void | readHeader (rownr_t nrrow, Bool firstTime) |
Read the header info. More... | |
void | updateRowMap (uInt cubeNr, uInt pos, rownr_t rownr) |
Update the map of row numbers to cube number plus offset. More... | |
void | extendRowMap (rownr_t nrow) |
Extend the map of row numbers to cube number plus offset will new empty entries. More... | |
Private Attributes | |
IPosition | defaultTileShape_p |
The default tile shape. More... | |
Block< uInt > | rowMap_p |
The map of row number to cube and position in cube. More... | |
Block< uInt > | cubeMap_p |
Block< uInt > | posMap_p |
uInt | nrUsedRowMap_p |
The nr of elements used in the map blocks. More... | |
Int | lastHC_p |
The last hypercube found. More... | |
Additional Inherited Members | |
Static Public Attributes inherited from casacore::DataManager | |
static rownr_t | MAXROWNR32 |
Define the highest row number that can be represented as signed 32-bit. More... | |
Protected Member Functions inherited from casacore::TiledStMan | |
void | setPersMaxCacheSize (uInt nMiB) |
Set the persistent maximum cache size (in MiB). More... | |
uInt | getBindings (const Vector< String > &columnNames, PtrBlock< TSMColumn * > &colSet, Bool mustExist) const |
Get the bindings of the columns with the given names. More... | |
virtual const TableDesc & | getDesc () const |
Get the table description needed for the hypercolumn description. More... | |
void | checkValues (const PtrBlock< TSMColumn * > &colSet, const Record &values) const |
Check if values are given in the record for all columns in the block. More... | |
void | checkCoordinates (const PtrBlock< TSMColumn * > &coordColSet, const IPosition &cubeShape, const Record &values) const |
Check if the coordinate values are correct. More... | |
void | checkShapeColumn (const IPosition &shape) const |
Check if the shapes of FixedShape data and coordinate columns match. More... | |
void | checkCoordinatesShapes (const TSMCube *hypercube, const IPosition &cubeShape) const |
Check if the cube shape matches that of defined coordinates. More... | |
void | checkAddHypercube (const IPosition &cubeShape, const Record &values) const |
Check if the hypercube to be added is correctly defined. More... | |
TSMCube * | makeHypercube (const IPosition &cubeShape, const IPosition &tileShape, const Record &values) |
Make a new TSMCube object. More... | |
Int | getCubeIndex (const Record &idValues) const |
Get the index of the hypercube with the given id-values. More... | |
rownr_t | addedNrrow (const IPosition &shape, uInt incrInLastDim) const |
Determine how many rows need to be added for an extension (in the last dimension) of a hypercube with the given shape. More... | |
Bool | flushCaches (Bool fsync) |
Flush the caches of all hypercubes. More... | |
AipsIO * | headerFileCreate () |
Create the TSM header file. More... | |
AipsIO * | headerFileOpen () |
Open the TSM header file. More... | |
void | headerFilePut (AipsIO &headerFile, uInt64 nrCube) |
Write the data into the header file. More... | |
uInt | headerFileGet (AipsIO &headerFile, rownr_t tabNrrow, Bool firstTime, Int extraNdim) |
Read the data from the header file. More... | |
void | headerFileClose (AipsIO *headerFile) |
Close the header file. More... | |
void | setup (Int extraNdim=-1) |
Set up the TiledStMan variables from the table description. More... | |
void | createFile (uInt index) |
Create a TSMFile object and store its pointer at the given index in the block. More... | |
int | arrayDataType (int dataType) const |
Convert the scalar data type to an array data type. More... | |
Protected Member Functions inherited from casacore::DataManager | |
void | decrementNcolumn () |
Decrement number of columns (in case a column is deleted). More... | |
void | setEndian (Bool bigEndian) |
Tell the data manager if big or little endian format is needed. More... | |
void | setTsmOption (const TSMOption &tsmOption) |
Tell the data manager which TSM option to use. More... | |
void | setMultiFile (MultiFileBase *mfile) |
Tell the data manager that MultiFile can be used. More... | |
void | throwDataTypeOther (const String &columnName, int dataType) const |
Throw an exception in case data type is TpOther, because the storage managers (and maybe other data managers) do not support such columns. More... | |
Protected Attributes inherited from casacore::TiledStMan | |
String | hypercolumnName_p |
The name of the hypercolumn. More... | |
rownr_t | nrrow_p |
The number of rows in the columns. More... | |
PtrBlock< TSMColumn * > | colSet_p |
The assembly of all columns. More... | |
PtrBlock< TSMDataColumn * > | dataCols_p |
The assembly of all data columns. More... | |
PtrBlock< TSMColumn * > | dataColSet_p |
PtrBlock< TSMColumn * > | idColSet_p |
The assembly of all id columns. More... | |
PtrBlock< TSMColumn * > | coordColSet_p |
The assembly of all coordinate columns. More... | |
PtrBlock< TSMFile * > | fileSet_p |
The assembly of all TSMFile objects. More... | |
PtrBlock< TSMCube * > | cubeSet_p |
The assembly of all TSMCube objects. More... | |
uInt | persMaxCacheSize_p |
The persistent maximum cache size (in MiB) for a hypercube. More... | |
uInt | maxCacheSize_p |
The actual maximum cache size for a hypercube (in MiB). More... | |
uInt | nrdim_p |
The dimensionality of the hypercolumn. More... | |
uInt | nrCoordVector_p |
The number of vector coordinates. More... | |
IPosition | fixedCellShape_p |
The fixed cell shape. More... | |
Bool | dataChanged_p |
Has any data changed since the last flush? More... | |
Tiled Data Storage Manager using the shape as id.
Public interface
TiledShapeStMan is the Tiled Storage Manager where the shape is used as id to support variable shaped arrays.
TiledShapeStMan is a derivation from TiledStMan, the abstract tiled storage manager class. A description of the basics of tiled storage managers is given in the Tables module description.
TiledShapeStMan creates a hypercube for each different shape of the data arrays. For example, if a table contains line and continuum data of an observation, it results in 2 hypercubes. TiledShapeStMan does it all automatically, so it is much easier to use than class TiledDataStMan.
TiledShapeStMan is meant for columns with not too many different shapes, otherwise looking for a matching hypercube may take too long. When many different shapes are used, class TiledCellStMan should be used instead.
TiledShapeStMan has the following (extra) properties:
TiledDataStMan proved to be very powerful, but also a bit cumbersome to use because a few special functions need to be called. TiledShapeStMan alleviates that problem.
Note that in this example the same shape is used for each row, but it could have been different.
Definition at line 183 of file TiledShapeStMan.h.
casacore::TiledShapeStMan::TiledShapeStMan | ( | const String & | hypercolumnName, |
const IPosition & | defaultTileShape, | ||
uInt64 | maximumCacheSize = 0 |
||
) |
Create a TiledShapeStMan storage manager for the hypercolumn with the given name.
The hypercolumn name is also the name of the storage manager. The given maximum cache size (default is unlimited) is persistent, thus will be reused when the table is read back. Note that the class ROTiledStManAccessor allows one to overwrite the maximum cache size temporarily.
The constructor taking a Record expects fields in the record with the name of the arguments in uppercase. If not defined, their default value is used.
casacore::TiledShapeStMan::~TiledShapeStMan | ( | ) |
|
private |
Create a TiledShapeStMan.
This constructor is private, because it should only be used by makeObject.
|
private |
Forbid copy constructor.
|
private |
Add a hypercube.
The number of rows in the table must be large enough to accommodate this hypercube. The possible id values must be given in the record, while coordinate values are optional. The field names in the record should match the coordinate and id column names. The last dimension in the cube shape can be zero, indicating that the hypercube is extensible.
|
privatevirtual |
Add rows to the storage manager.
Reimplemented from casacore::DataManager.
|
virtual |
TiledShapeStMan can access a column if there are 2 hypercubes and the first one is empty.
Reimplemented from casacore::TiledStMan.
|
virtual |
Clone this object.
It does not clone TSMColumn objects possibly used.
Implements casacore::DataManager.
|
privatevirtual |
Let the storage manager create files as needed for a new table.
This allows a column with an indirect array to create its file.
Reimplemented from casacore::DataManager.
|
virtual |
Return a record containing data manager specifications and info.
Reimplemented from casacore::TiledStMan.
|
virtual |
|
privatevirtual |
Get the default tile shape.
Reimplemented from casacore::TiledStMan.
Extend the hypercube with the given number of elements in the last dimension.
The record should contain the id values (to get the correct hypercube) and optionally coordinate values for the elements added.
|
private |
Extend the map of row numbers to cube number plus offset will new empty entries.
Find the hypercube for the given shape.
It returns -1 when not found.
Flush and optionally fsync the data.
It returns a True status if it had to flush (i.e. if data have changed).
Implements casacore::DataManager.
Get the hypercube in which the given row is stored.
Implements casacore::TiledStMan.
|
privatevirtual |
Get the hypercube in which the given row is stored.
It also returns the position of the row in that hypercube.
Implements casacore::TiledStMan.
|
static |
Make the object from the type name string.
This function gets registered in the DataManager "constructor" map.
|
private |
Forbid assignment.
Read the header info.
Implements casacore::TiledStMan.
|
virtual |
Set the shape and tile shape of the given hypercube.
It is used when the first row in a new hypercube is written. If needed it adds a dimension to the shape, which reflects the row dimension. The tile shape in that dimension is by default 1.
Reimplemented from casacore::TiledStMan.
|
privatevirtual |
Check if the hypercolumn definition fits this storage manager.
Reimplemented from casacore::TiledStMan.
|
virtual |
Test if only one hypercube is used by this storage manager.
If not, throw an exception. Otherwise return the hypercube.
Reimplemented from casacore::TiledStMan.
Update the map of row numbers to cube number plus offset.
Definition at line 312 of file TiledShapeStMan.h.
|
private |
The default tile shape.
Definition at line 309 of file TiledShapeStMan.h.
|
private |
The last hypercube found.
Definition at line 317 of file TiledShapeStMan.h.
|
private |
The nr of elements used in the map blocks.
Definition at line 315 of file TiledShapeStMan.h.
Definition at line 313 of file TiledShapeStMan.h.
The map of row number to cube and position in cube.
Definition at line 311 of file TiledShapeStMan.h.