casacore
|
Tiled hypercube in a table. More...
#include <TSMCube.h>
Public Types | |
enum | AccessType { NoAccess, CellAccess, SliceAccess, ColumnAccess, ColumnSliceAccess } |
Define the possible access types for TSMDataColumn. More... | |
Public Member Functions | |
TSMCube (TiledStMan *stman, TSMFile *file, const IPosition &cubeShape, const IPosition &tileShape, const Record &values, Int64 fileOffset, Bool useDerived=False) | |
Construct the hypercube using the given file with the given shape. More... | |
TSMCube (TiledStMan *stman, AipsIO &ios, Bool useDerived=False) | |
Reconstruct the hypercube by reading its data from the AipsIO stream. More... | |
virtual | ~TSMCube () |
virtual void | flushCache () |
Flush the data in the cache. More... | |
void | clearCache (Bool doFlush=True) |
Clear the cache, so data will be reread. More... | |
void | emptyCache () |
Empty the cache. More... | |
virtual void | showCacheStatistics (ostream &os) const |
Show the cache statistics. More... | |
void | putObject (AipsIO &ios) |
Put the data of the object into the AipsIO stream. More... | |
Int | getObject (AipsIO &ios) |
Get the data of the object from the AipsIO stream. More... | |
virtual void | resync (AipsIO &ios) |
Resync the object with the data file. More... | |
Bool | isExtensible () const |
Is the hypercube extensible? More... | |
uInt | bucketSize () const |
Get the bucket size (bytes). More... | |
uInt | localTileLength () const |
Get the length of a tile (in bytes) in local format. More... | |
virtual void | setShape (const IPosition &cubeShape, const IPosition &tileShape) |
Set the hypercube shape. More... | |
const IPosition & | cubeShape () const |
Get the shape of the hypercube. More... | |
const IPosition & | tileShape () const |
Get the shape of the tiles. More... | |
IPosition | cellShape () const |
Get the shape of the data cells in the cube. More... | |
uInt | coordinateSize (const String &coordinateName) const |
Get the size of a coordinate (i.e. More... | |
const Record & | valueRecord () const |
Get the record containing the id and coordinate values. More... | |
Record & | rwValueRecord () |
Bool | matches (const PtrBlock< TSMColumn * > &idColSet, const Record &idValues) |
Test if the id values match. More... | |
virtual void | extend (uInt64 nr, const Record &coordValues, const TSMColumn *lastCoordColumn) |
Extend the last dimension of the cube with the given number. More... | |
void | extendCoordinates (const Record &coordValues, const String &coordName, uInt length) |
Extend the coordinates vector for the given coordinate to the given length with the given coordValues. More... | |
virtual void | accessSection (const IPosition &start, const IPosition &end, char *section, uInt colnr, uInt localPixelSize, uInt externalPixelSize, Bool writeFlag) |
Read or write a section in the cube. More... | |
virtual void | accessStrided (const IPosition &start, const IPosition &end, const IPosition &stride, char *section, uInt colnr, uInt localPixelSize, uInt externalPixelSize, Bool writeFlag) |
Read or write a section in a strided way. More... | |
uInt | cacheSize () const |
Get the current cache size (in buckets). More... | |
uInt | calcCacheSize (const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath) const |
Calculate the cache size (in buckets) for the given slice and access path. More... | |
virtual void | setCacheSize (const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, Bool forceSmaller, Bool userSet) |
Set the cache size for the given slice and access path. More... | |
virtual void | setCacheSize (uInt cacheSize, Bool forceSmaller, Bool userSet) |
Resize the cache object. More... | |
uInt | validateCacheSize (uInt cacheSize) const |
Validate the cache size (in buckets). More... | |
Bool | userSetCache () const |
Determine if the user set the cache size (using setCacheSize). More... | |
AccessType | getLastColAccess () const |
Functions for TSMDataColumn to keep track of the last type of access to a hypercube. More... | |
const IPosition & | getLastColSlice () const |
void | setLastColAccess (AccessType type) |
void | setLastColSlice (const IPosition &slice) |
Static Public Member Functions | |
static uInt | calcCacheSize (const IPosition &cubeShape, const IPosition &tileShape, Bool extensible, const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, uInt maxCacheSizeMiB, uInt bucketSize) |
static uInt | validateCacheSize (uInt cacheSize, uInt maxSizeMiB, uInt bucketSize) |
Protected Member Functions | |
void | setup () |
Initialize the various variables. More... | |
void | setupNrTiles () |
IPosition | adjustTileShape (const IPosition &cubeShape, const IPosition &tileShape) const |
Adjust the tile shape to the hypercube shape. More... | |
void | resizeTileSections () |
Resize the IPosition member variables used in accessSection() if nrdim_p changes value. More... | |
Protected Attributes | |
char * | cachedTile_p |
TiledStMan * | stmanPtr_p |
Pointer to the parent storage manager. More... | |
Bool | useDerived_p |
Is the class used directly or only by a derived class only? More... | |
Record | values_p |
The values of the possible id and coordinate columns. More... | |
Bool | extensible_p |
Is the hypercube extensible? More... | |
uInt | nrdim_p |
Dimensionality of the hypercube. More... | |
uInt | nrTiles_p |
Number of tiles in the hypercube. More... | |
IPosition | cubeShape_p |
The shape of the hypercube. More... | |
IPosition | tileShape_p |
The shape of the tiles in the hypercube. More... | |
IPosition | tilesPerDim_p |
The number of tiles in each hypercube dimension. More... | |
TSMShape | expandedTileShape_p |
Precomputed tileShape information. More... | |
TSMShape | expandedTilesPerDim_p |
Precomputed tilesPerDim information. More... | |
uInt | nrTilesSubCube_p |
Number of tiles in all but last dimension (used when extending). More... | |
uInt | tileSize_p |
The tilesize in bytes. More... | |
TSMFile * | filePtr_p |
Pointer to the TSMFile object holding the data. More... | |
Int64 | fileOffset_p |
Offset in the TSMFile object where the data of this hypercube starts. More... | |
Block< uInt > | externalOffset_p |
Offset for each data column in a tile (in external format). More... | |
Block< uInt > | localOffset_p |
Offset for each data column in a tile (in local format). More... | |
uInt | bucketSize_p |
The bucket size in bytes (is equal to tile size in bytes). More... | |
uInt | localTileLength_p |
The tile size in bytes in local format. More... | |
BucketCache * | cache_p |
The bucket cache. More... | |
Bool | userSetCache_p |
Did the user set the cache size? More... | |
AccessType | lastColAccess_p |
Was the last column access to a cell, slice, or column? More... | |
IPosition | lastColSlice_p |
The slice shape of the last column access to a slice. More... | |
IPosition | nrTileSection_p |
IPosition variables used in accessSection(); declared here as member variables to avoid significant construction and desctruction overhead if they are local to accessSection() #tiles needed for the section. More... | |
IPosition | startTile_p |
First tile needed. More... | |
IPosition | endTile_p |
Last tile needed. More... | |
IPosition | startPixelInFirstTile_p |
First pixel in first tile. More... | |
IPosition | endPixelInFirstTile_p |
Last pixel in first tile. More... | |
IPosition | endPixelInLastTile_p |
Last pixel in last tile. More... | |
Private Member Functions | |
TSMCube (const TSMCube &) | |
Forbid copy constructor. More... | |
TSMCube & | operator= (const TSMCube &) |
Forbid assignment. More... | |
BucketCache * | getCache () |
Get the cache object. More... | |
virtual void | makeCache () |
Construct the cache object (if not constructed yet). More... | |
virtual void | resyncCache () |
Resync the cache object. More... | |
virtual void | deleteCache () |
Delete the cache object. More... | |
void | accessLine (char *section, uInt pixelOffset, uInt localPixelSize, Bool writeFlag, BucketCache *cachePtr, const IPosition &startTile, uInt endTile, const IPosition &startPixelInFirstTile, uInt endPixelInLastTile, uInt lineIndex) |
Access a line in a more optimized way. More... | |
char * | readTile (const char *external) |
Define the functions doing the actual read and write of the data in the tile and converting it to/from local format. More... | |
void | writeTile (char *external, const char *local) |
Static Private Member Functions | |
static char * | readCallBack (void *owner, const char *external) |
Define the callback functions for the BucketCache. More... | |
static void | writeCallBack (void *owner, char *external, const char *local) |
static char * | initCallBack (void *owner) |
static void | deleteCallBack (void *owner, char *buffer) |
Tiled hypercube in a table.
Internal
TSMCube represents a hypercube in the Tiled Storage Manager.
TSMCube defines a tiled hypercube. The data is stored in a TSMFile object and accessed using a BucketCache object. The hypercube can be extensible in its last dimension to support tables with a size which is not known in advance.
Normally hypercubes share the same TSMFile object, but extensible hypercubes have their own TSMFile object (to be extensible). If the hypercolumn has multiple data columns, their cells share the same tiles. Per tile data column A appears first, thereafter B, etc..
The data in the cache is held in external format and is converted when accessed. The alternative would be to hold it in the cache in local format and convert it when read/written from the file. It was felt that the latter approach would generate more needless conversions.
The possible id and coordinate values are stored in a Record object. They are written in the main hypercube AipsIO file.
TSMCube uses the maximum cache size set for a Tiled Storage manager. The description of class ROTiledStManAccessor contains a discussion about the effect of setting the maximum cache size.
TSMCube encapsulates all operations on a hypercube.
Define the possible access types for TSMDataColumn.
Enumerator | |
---|---|
NoAccess | |
CellAccess | |
SliceAccess | |
ColumnAccess | |
ColumnSliceAccess |
casacore::TSMCube::TSMCube | ( | TiledStMan * | stman, |
TSMFile * | file, | ||
const IPosition & | cubeShape, | ||
const IPosition & | tileShape, | ||
const Record & | values, | ||
Int64 | fileOffset, | ||
Bool | useDerived = False |
||
) |
Construct the hypercube using the given file with the given shape.
The record contains the id and possible coordinate values.
If the cubeshape is empty, the hypercube is still undefined and can be added later with setShape. That is only used by TiledCellStMan.
The fileOffset argument is meant for class TiledFileAccess.
casacore::TSMCube::TSMCube | ( | TiledStMan * | stman, |
AipsIO & | ios, | ||
Bool | useDerived = False |
||
) |
|
virtual |
|
private |
Forbid copy constructor.
|
private |
Access a line in a more optimized way.
|
virtual |
Read or write a section in the cube.
It is assumed that the section buffer is long enough.
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
|
virtual |
Read or write a section in a strided way.
It is assumed that the section buffer is long enough.
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
|
protected |
Adjust the tile shape to the hypercube shape.
A size of 0 gets set to 1. A tile size > cube size gets set to the cube size.
|
inline |
Get the bucket size (bytes).
It is the length of a tile in external format.
Definition at line 430 of file TSMCube.h.
References bucketSize_p.
uInt casacore::TSMCube::cacheSize | ( | ) | const |
Get the current cache size (in buckets).
Referenced by casacore::TiledFileAccess::cacheSize().
uInt casacore::TSMCube::calcCacheSize | ( | const IPosition & | sliceShape, |
const IPosition & | windowStart, | ||
const IPosition & | windowLength, | ||
const IPosition & | axisPath | ||
) | const |
Calculate the cache size (in buckets) for the given slice and access path.
|
static |
IPosition casacore::TSMCube::cellShape | ( | ) | const |
Get the shape of the data cells in the cube.
Clear the cache, so data will be reread.
If wanted, the data is flushed before the cache is cleared.
Get the size of a coordinate (i.e.
the number of values in it). If not defined, it returns zero.
|
inline |
Get the shape of the hypercube.
Definition at line 438 of file TSMCube.h.
References cubeShape_p.
Referenced by casacore::TiledFileAccess::shape().
|
privatevirtual |
Delete the cache object.
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
|
staticprivate |
void casacore::TSMCube::emptyCache | ( | ) |
Empty the cache.
It will flush the cache as needed and remove all buckets from it resulting in a possibly large drop in memory used. It'll also clear the userSetCache_p
flag.
Referenced by casacore::TiledFileAccess::clearCache().
|
virtual |
Extend the last dimension of the cube with the given number.
The record can contain the coordinates of the elements added.
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
void casacore::TSMCube::extendCoordinates | ( | const Record & | coordValues, |
const String & | coordName, | ||
uInt | length | ||
) |
Extend the coordinates vector for the given coordinate to the given length with the given coordValues.
It will be initialized to zero if no coordValues are given. If the coordinate vector does not exist yet, it will be created.
|
virtual |
Flush the data in the cache.
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
Referenced by casacore::TiledFileAccess::flush().
|
inlineprivate |
Get the cache object.
This will construct the cache object if not present yet.
Definition at line 423 of file TSMCube.h.
References cache_p, and makeCache().
|
inline |
Functions for TSMDataColumn to keep track of the last type of access to a hypercube.
It uses it to determine if the cache has to be reset.
Definition at line 458 of file TSMCube.h.
References lastColAccess_p.
|
inline |
Definition at line 462 of file TSMCube.h.
References lastColSlice_p.
Get the data of the object from the AipsIO stream.
It returns the data manager sequence number, which is -1 if no file is attached to the cube (for cells without a value).
|
staticprivate |
Bool casacore::TSMCube::isExtensible | ( | ) | const |
Is the hypercube extensible?
|
inline |
Get the length of a tile (in bytes) in local format.
Definition at line 434 of file TSMCube.h.
References localTileLength_p.
|
privatevirtual |
Construct the cache object (if not constructed yet).
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
Referenced by getCache().
Bool casacore::TSMCube::matches | ( | const PtrBlock< TSMColumn * > & | idColSet, |
const Record & | idValues | ||
) |
Test if the id values match.
void casacore::TSMCube::putObject | ( | AipsIO & | ios | ) |
Put the data of the object into the AipsIO stream.
|
staticprivate |
Define the callback functions for the BucketCache.
|
private |
Define the functions doing the actual read and write of the data in the tile and converting it to/from local format.
|
protected |
Resize the IPosition member variables used in accessSection() if nrdim_p changes value.
|
virtual |
Resync the object with the data file.
It reads the object, and adjusts the cache.
|
privatevirtual |
Resync the cache object.
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
|
inline |
|
virtual |
Set the cache size for the given slice and access path.
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
Referenced by casacore::TiledFileAccess::setCacheSize().
|
virtual |
Resize the cache object.
If forceSmaller is False, the cache will only be resized when it grows. If the given size exceeds the maximum size with more than 10%, the maximum size will be used. The cacheSize has to be given in buckets.
The flag userSet
inidicates if the cache size is set by the user (by an Accessor object) or automatically (by TSMDataColumn).
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
|
inline |
Definition at line 466 of file TSMCube.h.
References lastColAccess_p.
|
inline |
Definition at line 470 of file TSMCube.h.
References lastColSlice_p, casacore::IPosition::nelements(), and casacore::IPosition::resize().
|
virtual |
Set the hypercube shape.
This is only possible if the shape was not defined yet.
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
|
protected |
Initialize the various variables.
|
protected |
|
virtual |
Show the cache statistics.
Reimplemented in casacore::TSMCubeMMap, and casacore::TSMCubeBuff.
Referenced by casacore::TiledFileAccess::showCacheStatistics().
|
inline |
Get the shape of the tiles.
Definition at line 442 of file TSMCube.h.
References tileShape_p.
Referenced by casacore::TiledFileAccess::tileShape().
|
inline |
Determine if the user set the cache size (using setCacheSize).
Definition at line 454 of file TSMCube.h.
References userSetCache_p.
Validate the cache size (in buckets).
This means it will return the given cache size (in buckets) if smaller than the maximum cache size (given in MiB). Otherwise the maximum is returned.
|
static |
|
inline |
Get the record containing the id and coordinate values.
It is used by TSMIdColumn and TSMCoordColumn.
Definition at line 446 of file TSMCube.h.
References values_p.
|
staticprivate |
|
private |
|
protected |
The bucket size in bytes (is equal to tile size in bytes).
Definition at line 392 of file TSMCube.h.
Referenced by bucketSize().
|
protected |
|
protected |
|
protected |
|
protected |
|
protected |
|
protected |
|
protected |
|
protected |
|
protected |
|
protected |
|
protected |
Was the last column access to a cell, slice, or column?
Definition at line 400 of file TSMCube.h.
Referenced by getLastColAccess(), and setLastColAccess().
|
protected |
The slice shape of the last column access to a slice.
Definition at line 402 of file TSMCube.h.
Referenced by getLastColSlice(), and setLastColSlice().
|
protected |
The tile size in bytes in local format.
Definition at line 394 of file TSMCube.h.
Referenced by localTileLength().
|
protected |
|
protected |
|
protected |
IPosition variables used in accessSection(); declared here as member variables to avoid significant construction and desctruction overhead if they are local to accessSection() #tiles needed for the section.
|
protected |
|
protected |
|
protected |
|
protected |
|
protected |
The shape of the tiles in the hypercube.
Definition at line 372 of file TSMCube.h.
Referenced by tileShape().
|
protected |
|
protected |
|
protected |
|
protected |
Did the user set the cache size?
Definition at line 398 of file TSMCube.h.
Referenced by userSetCache().
|
protected |
The values of the possible id and coordinate columns.
Definition at line 362 of file TSMCube.h.
Referenced by rwValueRecord(), and valueRecord().