Tiled Column Storage Manager. More...

#include <TiledColumnStMan.h>

Inheritance diagram for casacore::TiledColumnStMan:

Public Member Functions
	TiledColumnStMan (const String &hypercolumnName, const IPosition &tileShape, uInt64 maximumCacheSize=0)
	Create a TiledDataStMan storage manager for the hypercolumn with the given name. More...

	TiledColumnStMan (const String &hypercolumnName, const Record &spec)

	~TiledColumnStMan ()

virtual DataManager *	clone () const
	Clone this object. More...

virtual Bool	canAccessColumn () const
	TiledColumnStMan can always access a column. More...

virtual String	dataManagerType () const
	Get the type name of the data manager (i.e. More...

Public Member Functions inherited from casacore::TiledStMan
	TiledStMan ()
	Create a TiledStMan. More...

	TiledStMan (const String &hypercolumnName, uInt maximumCacheSizeMiB)
	Create a TiledStMan storage manager. More...

virtual	~TiledStMan ()

virtual String	dataManagerName () const
	Get the name given to the storage manager. More...

void	setDataManagerName (const String &newHypercolumnName)

virtual Record	dataManagerSpec () const
	Return a record containing data manager specifications. More...

virtual Record	getProperties () const
	Get data manager properties that can be modified. More...

virtual void	setProperties (const Record &spec)
	Modify data manager properties. More...

void	setDataChanged ()
	Set the flag to "data has changed since last flush". More...

virtual void	setMaximumCacheSize (uInt nMiB)
	Set the maximum cache size (in MiB) in a non-persistent way. More...

uInt	maximumCacheSize () const
	Get the current maximum cache size (in MiB (MibiByte)). More...

uInt	cacheSize (rownr_t rownr) const
	Get the current cache size (in buckets) for the hypercube in the given row. More...

const IPosition &	hypercubeShape (rownr_t rownr) const
	Get the hypercube shape of the data in the given row. More...

const IPosition &	tileShape (rownr_t rownr) const
	Get the tile shape of the data in the given row. More...

uInt64	bucketSize (rownr_t rownr) const
	Get the bucket size (in bytes) of the hypercube in the given row. More...

virtual Bool	canChangeShape () const
	Can the tiled storage manager handle changing array shapes? The default is no (but TiledCellStMan can). More...

virtual Bool	hasMultiFileSupport () const
	The data manager supports use of MultiFile. More...

uInt	calcCacheSize (rownr_t rownr, const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath) const
	Calculate the cache size (in buckets) for accessing the hypercube containing the given row. More...

void	setCacheSize (rownr_t rownr, const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, Bool forceSmaller)
	Set the cache size using the `calcCacheSize` function mentioned above. More...

void	setCacheSize (rownr_t rownr, uInt nbuckets, Bool forceSmaller)
	Set the cache size for accessing the hypercube containing the given row. More...

void	setHypercubeCacheSize (uInt hypercube, uInt nbuckets, Bool forceSmaller)
	Sets the cache size using the hypercube instead of the row number. More...

Bool	userSetCache (rownr_t rownr) const
	Determine if the user set the cache size (using setCacheSize). More...

void	emptyCaches ()
	Empty the caches used by the hypercubes in this storage manager. More...

void	showCacheStatistics (ostream &os) const
	Show the statistics of all caches used. More...

uInt64	getLengthOffset (uInt64 nrPixels, Block< uInt > &dataOffset, Block< uInt > &localOffset, uInt &localTileLength) const
	Get the length of the data for the given number of pixels. More...

uInt	nrCoordVector () const
	Get the number of coordinate vectors. More...

rownr_t	nrow () const
	Get the nr of rows in this storage manager. More...

Bool	canAddRow () const
	Does the storage manager allow to add rows? (yes) More...

uInt	nhypercubes () const
	Return the number of hypercubes. More...

virtual TSMCube *	singleHypercube ()
	Test if only one hypercube is used by this storage manager. More...

const TSMCube *	getTSMCube (uInt hypercube) const
	Get the given hypercube. More...

TSMCube *	getTSMCube (uInt hypercube)

const TSMCube *	getHypercube (rownr_t rownr) const
	Get the hypercube in which the given row is stored. More...

TSMCube *	makeTSMCube (TSMFile *file, const IPosition &cubeShape, const IPosition &tileShape, const Record &values, Int64 fileOffset=-1)
	Make the correct TSMCube type (depending on tsmOption()). More...

void	readTile (char local, const Block< uInt > &localOffset, const char external, const Block< uInt > &externalOffset, uInt nrpixels)
	Read a tile and convert the data to local format. More...

void	writeTile (char external, const Block< uInt > &externalOffset, const char local, const Block< uInt > &localOffset, uInt nrpixels)
	Write a tile after converting the data to external format. More...

TSMFile *	getFile (uInt sequenceNumber)
	Get the TSMFile object with the given sequence number. More...

virtual rownr_t	open64 (rownr_t nrrow, AipsIO &)
	Open the storage manager for an existing table. More...

virtual rownr_t	resync64 (rownr_t nrrow)
	Resync the storage manager with the new file contents. More...

virtual void	reopenRW ()
	Reopen all files used in this storage manager for read/write access. More...

virtual void	deleteManager ()
	The data manager will be deleted (because all its columns are requested to be deleted). More...

DataManagerColumn *	makeScalarColumn (const String &name, int dataType, const String &dataTypeID)
	Create a column in the storage manager on behalf of a table column. More...

DataManagerColumn *	makeDirArrColumn (const String &name, int dataType, const String &dataTypeID)
	Create a direct array column. More...

DataManagerColumn *	makeIndArrColumn (const String &name, int dataType, const String &dataTypeID)
	Create an indirect array column. More...

Bool	canReallocateColumns () const
	The TiledStMan wants to do reallocateColumn. More...

DataManagerColumn *	reallocateColumn (DataManagerColumn *column)
	Reallocate the column object if it is part of this data manager. More...

virtual void	setShape (rownr_t rownr, TSMCube *hypercube, const IPosition &shape, const IPosition &tileShape)
	Set the shape and tile shape of a hypercube. More...

void	checkCubeShape (const TSMCube *hypercube, const IPosition &cubeShape) const
	Check the shape to be set for a hypercube. More...

int	coordinateDataType (const String &columnName) const
	Get the data type of the coordinate column with the given name. More...

void	initCoordinates (TSMCube *hypercube)
	Initialize the new coordinates for the given cube. More...

const TSMDataColumn *	getDataColumn (uInt colnr) const
	Get pointer to data column object. More...

Public Member Functions inherited from casacore::DataManager
	DataManager ()
	Default constructor. More...

virtual	~DataManager ()

void	dataManagerInfo (Record &info) const
	Add SEQNR and SPEC (the DataManagerSpec subrecord) to the info. More...

virtual Bool	isStorageManager () const
	Is the data manager a storage manager? The default is yes. More...

uInt	sequenceNr () const
	Get the (unique) sequence nr of this data manager. More...

uInt	ncolumn () const
	Get the nr of columns in this data manager (can be zero). More...

Bool	asBigEndian () const
	Have the data to be stored in big or little endian canonical format? More...

const TSMOption &	tsmOption () const
	Get the TSM option. More...

MultiFileBase *	multiFile ()
	Get the MultiFile pointer (can be 0). More...

String	keywordName (const String &keyword) const
	Compose a keyword name from the given keyword appended with the sequence number (e.g. More...

String	fileName () const
	Compose a unique filename from the table name and sequence number. More...

ByteIO::OpenOption	fileOption () const
	Get the AipsIO option of the underlying file. More...

virtual Bool	isRegular () const
	Is this a regular storage manager? It is regular if it allows addition of rows and writing data in them. More...

Table &	table () const
	Get the table this object is associated with. More...

virtual Bool	canRemoveRow () const
	Does the data manager allow to delete rows? (default no) More...

virtual Bool	canAddColumn () const
	Does the data manager allow to add columns? (default no) More...

virtual Bool	canRemoveColumn () const
	Does the data manager allow to delete columns? (default no) More...

virtual Bool	canRenameColumn () const
	Does the data manager allow to rename columns? (default yes) More...

virtual void	showCacheStatistics (std::ostream &) const
	Show the data manager's IO statistics. More...

DataManagerColumn *	createScalarColumn (const String &columnName, int dataType, const String &dataTypeId)
	Create a column in the data manager on behalf of a table column. More...

DataManagerColumn *	createDirArrColumn (const String &columnName, int dataType, const String &dataTypeId)
	Create a direct array column. More...

DataManagerColumn *	createIndArrColumn (const String &columnName, int dataType, const String &dataTypeId)
	Create an indirect array column. More...

DataManager *	getClone () const
	Has the object already been cloned? More...

void	setClone (DataManager *clone) const
	Set the pointer to the clone. More...

Static Public Member Functions
static DataManager *	makeObject (const String &dataManagerType, const Record &spec)
	Make the object from the type name string. More...

Static Public Member Functions inherited from casacore::TiledStMan
static IPosition	makeTileShape (const IPosition &hypercubeShape, Double tolerance=0.5, uInt64 maxNrPixelsPerTile=4 1024 1024)
	Derive the tile shape from the hypercube shape for the given number of pixels per tile. More...

static IPosition	makeTileShape (const IPosition &hypercubeShape, const Vector< double > &weight, const Vector< double > &tolerance, uInt64 maxNrPixelsPerTile=4 1024 1024)

Static Public Member Functions inherited from casacore::DataManager
static void	registerCtor (const String &type, DataManagerCtor func)
	Register a mapping of a data manager type to its static construction function. More...

static DataManagerCtor	getCtor (const String &dataManagerType)
	Get the "constructor" of a data manager (thread-safe). More...

static Bool	isRegistered (const String &dataManagerType)
	Test if a data manager is registered (thread-safe). More...

static DataManager *	unknownDataManager (const String &dataManagerType, const Record &spec)
	Serve as default function for theirRegisterMap, which catches all unknown data manager types. More...

Private Member Functions
	TiledColumnStMan ()
	Create a TiledColumnStMan. More...

	TiledColumnStMan (const TiledColumnStMan &)
	Forbid copy constructor. More...

TiledColumnStMan &	operator= (const TiledColumnStMan &)
	Forbid assignment. More...

virtual IPosition	defaultTileShape () const
	Get the (default) tile shape. More...

void	addRow64 (rownr_t nrrow)
	Add rows to the storage manager. More...

virtual TSMCube *	getHypercube (rownr_t rownr)
	Get the hypercube in which the given row is stored. More...

virtual TSMCube *	getHypercube (rownr_t rownr, IPosition &position)
	Get the hypercube in which the given row is stored. More...

virtual void	setupCheck (const TableDesc &tableDesc, const Vector< String > &dataNames) const
	Check if the hypercolumn definition fits this storage manager. More...

virtual Bool	flush (AipsIO &, Bool fsync)
	Flush and optionally fsync the data. More...

virtual void	create64 (rownr_t nrrow)
	Let the storage manager create files as needed for a new table. More...

virtual void	readHeader (rownr_t nrrow, Bool firstTime)
	Read the header info. More...

Private Attributes
IPosition	tileShape_p

Additional Inherited Members
Static Public Attributes inherited from casacore::DataManager
static rownr_t	MAXROWNR32
	Define the highest row number that can be represented as signed 32-bit. More...

Protected Member Functions inherited from casacore::TiledStMan
void	setPersMaxCacheSize (uInt nMiB)
	Set the persistent maximum cache size (in MiB). More...

uInt	getBindings (const Vector< String > &columnNames, PtrBlock< TSMColumn * > &colSet, Bool mustExist) const
	Get the bindings of the columns with the given names. More...

virtual const TableDesc &	getDesc () const
	Get the table description needed for the hypercolumn description. More...

void	checkValues (const PtrBlock< TSMColumn * > &colSet, const Record &values) const
	Check if values are given in the record for all columns in the block. More...

void	checkCoordinates (const PtrBlock< TSMColumn * > &coordColSet, const IPosition &cubeShape, const Record &values) const
	Check if the coordinate values are correct. More...

void	checkShapeColumn (const IPosition &shape) const
	Check if the shapes of FixedShape data and coordinate columns match. More...

void	checkCoordinatesShapes (const TSMCube *hypercube, const IPosition &cubeShape) const
	Check if the cube shape matches that of defined coordinates. More...

void	checkAddHypercube (const IPosition &cubeShape, const Record &values) const
	Check if the hypercube to be added is correctly defined. More...

TSMCube *	makeHypercube (const IPosition &cubeShape, const IPosition &tileShape, const Record &values)
	Make a new TSMCube object. More...

Int	getCubeIndex (const Record &idValues) const
	Get the index of the hypercube with the given id-values. More...

rownr_t	addedNrrow (const IPosition &shape, uInt incrInLastDim) const
	Determine how many rows need to be added for an extension (in the last dimension) of a hypercube with the given shape. More...

Bool	flushCaches (Bool fsync)
	Flush the caches of all hypercubes. More...

AipsIO *	headerFileCreate ()
	Create the TSM header file. More...

AipsIO *	headerFileOpen ()
	Open the TSM header file. More...

void	headerFilePut (AipsIO &headerFile, uInt64 nrCube)
	Write the data into the header file. More...

uInt	headerFileGet (AipsIO &headerFile, rownr_t tabNrrow, Bool firstTime, Int extraNdim)
	Read the data from the header file. More...

void	headerFileClose (AipsIO *headerFile)
	Close the header file. More...

void	setup (Int extraNdim=-1)
	Set up the TiledStMan variables from the table description. More...

void	createFile (uInt index)
	Create a TSMFile object and store its pointer at the given index in the block. More...

int	arrayDataType (int dataType) const
	Convert the scalar data type to an array data type. More...

Protected Member Functions inherited from casacore::DataManager
void	decrementNcolumn ()
	Decrement number of columns (in case a column is deleted). More...

void	setEndian (Bool bigEndian)
	Tell the data manager if big or little endian format is needed. More...

void	setTsmOption (const TSMOption &tsmOption)
	Tell the data manager which TSM option to use. More...

void	setMultiFile (MultiFileBase *mfile)
	Tell the data manager that MultiFile can be used. More...

void	throwDataTypeOther (const String &columnName, int dataType) const
	Throw an exception in case data type is TpOther, because the storage managers (and maybe other data managers) do not support such columns. More...

Protected Attributes inherited from casacore::TiledStMan
String	hypercolumnName_p
	The name of the hypercolumn. More...

rownr_t	nrrow_p
	The number of rows in the columns. More...

PtrBlock< TSMColumn * >	colSet_p
	The assembly of all columns. More...

PtrBlock< TSMDataColumn * >	dataCols_p
	The assembly of all data columns. More...

PtrBlock< TSMColumn * >	dataColSet_p

PtrBlock< TSMColumn * >	idColSet_p
	The assembly of all id columns. More...

PtrBlock< TSMColumn * >	coordColSet_p
	The assembly of all coordinate columns. More...

PtrBlock< TSMFile * >	fileSet_p
	The assembly of all TSMFile objects. More...

PtrBlock< TSMCube * >	cubeSet_p
	The assembly of all TSMCube objects. More...

uInt	persMaxCacheSize_p
	The persistent maximum cache size (in MiB) for a hypercube. More...

uInt	maxCacheSize_p
	The actual maximum cache size for a hypercube (in MiB). More...

uInt	nrdim_p
	The dimensionality of the hypercolumn. More...

uInt	nrCoordVector_p
	The number of vector coordinates. More...

IPosition	fixedCellShape_p
	The fixed cell shape. More...

Bool	dataChanged_p
	Has any data changed since the last flush? More...

Detailed Description

Tiled Column Storage Manager.

Intended use:

Public interface

Review Status

Reviewed By:: UNKNOWN
Date Reviewed:: before2004/08/25

Prerequisite

TiledStMan
TSMCube
ROTiledStManAccessor for a discussion of the maximum cache size

Etymology

TiledColumnStMan is the Tiled Storage Manager storing an entire column as one hypercube.

Synopsis

TiledColumnStMan is a derivation from TiledStMan, the abstract tiled storage manager class. A description of the basics of tiled storage managers is given in the Tables module description.

TiledColumnStMan allows the user to create a tiled hypercube for an entire data column and extend it in an automatic way. It is meant to be used for fixed shaped data which have to be accessed in various directions.

The TiledColumnStMan has the following (extra) properties:

Addition of a row results in the extension of the hypercube. The data cells in all rows have to have the same shape. Therefore the columns stored by a TiledColumnStMan storage manager have to be fixed shaped (i.e. FixedShape attribute set in their column descriptions).
Coordinates for the hypercubes can be defined and (of course) their shapes have to match the hypercube shape. Their values have to be put explicitly (so it is not possible to define them via an extendHypercube call like in TiledDataStMan).
The tile shape of the hypercube has to be defined by means of the TiledColumnStMan constructor.

Motivation

This tiled storage manager does not require any special action (like calling add/extendHypercube) when used with a column containing equally shaped arrays.

Example

// Define the table description and the columns in it.
TableDesc td ("", "1", TableDesc::Scratch);
td.addColumn (ArrayColumnDesc<float>  ("RA", 1));
td.addColumn (ArrayColumnDesc<float>  ("Dec", 1));
td.addColumn (ScalarColumnDesc<float> ("Velocity"));
td.addColumn (ArrayColumnDesc<float>  ("Image", 2));
// Define the 3-dim hypercolumn with its data and coordinate columns.
// Note that its dimensionality must be one higher than the dimensionality
// of the data cells.
td.defineHypercolumn ("TSMExample",
3,
stringToVector ("Image"),
stringToVector ("RA,Dec,Velocity"));
// Now create a new table from the description.
SetupNewTable newtab("tTiledColumnStMan_tmp.data", td, Table::New);
// Create a TiledColumnStMan storage manager for the hypercolumn
// and bind the columns to it.
// The tile shape has to be specified for the storage manager.
TiledColumnStMan sm1 ("TSMExample", IPosition(3,16,32,32));
newtab.bindAll (sm1);
// Create the table.
Table table(newtab);
// Define the values for the coordinates of the hypercube.
Vector<float> raValues(512);
Vector<float> DecValues(512);
indgen (raValues);
indgen (decValues, float(100));
ArrayColumn<float> ra (table, "RA");
ArrayColumn<float> dec (table, "Dec");
ScalarColumn<float> velocity (table, "Velocity");
ArrayColumn<float> image (table, "Image");
Cube<float> imageValues(IPosition(2,512,512));
indgen (imageValues);
// Write some data into the data columns.
for (uInt i=0; i<64; i++) {
table.addRow();
image.put (i, imageValues);
// The RA and Dec have to be put only once, because they
// are the same for each row.
if (i == 0) {
ra.put (i, raValues);
dec.put (i, decValues);
}
velocity.put (i, float(i));
}

Definition at line 153 of file TiledColumnStMan.h.

Constructor & Destructor Documentation

casacore::TiledColumnStMan::TiledColumnStMan	(	const String &	hypercolumnName,
		const IPosition &	tileShape,
		uInt64	maximumCacheSize = `0`
	)

Create a TiledDataStMan storage manager for the hypercolumn with the given name.

The columns used should have the FixedShape attribute set. The hypercolumn name is also the name of the storage manager. The given tile shape will be used. The given maximum cache size in bytes (default is unlimited) is persistent, thus will be reused when the table is read back. Note that the class ROTiledStManAccessor allows one to overwrite the maximum cache size temporarily. Its description contains a discussion about the effects of setting a maximum cache.
The constructor taking a Record expects fields in the record with the name of the arguments in uppercase. If not defined, their default value is used.