TiledStMan.h

Go to the documentation of this file.

//# TiledStMan.h: Base class for Tiled Storage Managers
//# Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003
//# Associated Universities, Inc. Washington DC, USA.
//#
//# This library is free software; you can redistribute it and/or modify it
//# under the terms of the GNU Library General Public License as published by
//# the Free Software Foundation; either version 2 of the License, or (at your
//# option) any later version.
//#
//# This library is distributed in the hope that it will be useful, but WITHOUT
//# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
//# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
//# License for more details.
//#
//# You should have received a copy of the GNU Library General Public License
//# along with this library; if not, write to the Free Software Foundation,
//# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
//#
//# Correspondence concerning AIPS++ should be addressed as follows:
//#        Internet email: aips2-request@nrao.edu.
//#        Postal address: AIPS++ Project Office
//#                        National Radio Astronomy Observatory
//#                        520 Edgemont Road
//#                        Charlottesville, VA 22903-2475 USA
//#
//# $Id$

#ifndef TABLES_TILEDSTMAN_H
#define TABLES_TILEDSTMAN_H


//# Includes
#include <casacore/casa/aips.h>
#include <casacore/tables/DataMan/DataManager.h>
#include <casacore/casa/Containers/Block.h>
#include <casacore/casa/Arrays/ArrayFwd.h>
#include <casacore/casa/Arrays/IPosition.h>
#include <casacore/casa/OS/Conversion.h>
#include <casacore/casa/BasicSL/String.h>

namespace casacore { //# NAMESPACE CASACORE - BEGIN

//# Forward Declarations
class TSMColumn;
class TSMDataColumn;
class TSMCube;
class TSMFile;
class TableDesc;
class Record;

// <summary>
// Base class for Tiled Storage Manager classes
// </summary>

// <use visibility=export>

// <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
// </reviewed>

// <prerequisite>
//# Classes you should understand before using this one.
//   <li> Description of Tiled Storage Manager in module file
//        <linkto module=Tables:TiledStMan>Tables.h</linkto>
//   <li> <linkto class=DataManager>DataManager</linkto>
//   <li> <linkto class=TSMColumn>TSMColumn</linkto>
//   <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
//        for a discussion of the maximum cache size
// </prerequisite>

// <synopsis> 
// TiledStMan is the base class for Tiled Storage Managers.
// A tiled storage manager is capable of storing a hypercolumn
// (as defined by <linkto file="TableDesc.h#defineHypercolumn">
// TableDesc::defineHypercolumn</linkto>)
// in one or more hypercubes.
// <br>It is not necessary to define a hypercolumn. If not defined,
// it is assumed that all columns bound to this storage manager are
// data columns. At least one of the columns must have a fixed
// dimensionality and is used to determine the hypercube dimnensionality.
// <br>The general concept of these storage managers is explained in the
// <linkto module="Tables:TiledStMan">Tables module description</linkto>.
// <p>
// TiledStMan contains all common functions for the different tiled
// storage managers. In particular, it contains functions
// to check if the definitions of the shapes of hypercubes, coordinates, and
// data cells are consistent.
// It also contains various data members and functions to make them
// persistent by writing them into an AipsIO stream.
// </synopsis> 

// <motivation>
// This base class contains the common functionality of all
// tiled storage managers. The base class is still abstract.
// Only concrete tiled storage managers derived from it can
// be instantiated.
// <p>
// Tiled storage managers make access to array data possible with
// more or less the same efficiency for access along different axes.
// </motivation>

//# <todo asof="$DATE:$">
//# A List of bugs, limitations, extensions or planned refinements.
//# </todo>


class TiledStMan : public DataManager
{
public:
    // Create a TiledStMan.
    TiledStMan();

    // Create a TiledStMan storage manager.
    // The given maximum cache size (in MibiByte) is persistent,
    // thus will be reused when the table is read back. Note that the class
    // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
    // allows one to overwrite the maximum cache size temporarily.
    // Its description contains a discussion about the effects of
    // setting a maximum cache.
    TiledStMan (const String& hypercolumnName, uInt maximumCacheSizeMiB);

    virtual ~TiledStMan();

    // Get the name given to the storage manager.
    // This is the name of the hypercolumn.
    virtual String dataManagerName() const;

    void setDataManagerName (const String& newHypercolumnName);

    // Return a record containing data manager specifications.
    virtual Record dataManagerSpec() const;

    // Get data manager properties that can be modified.
    // It is only MaxCacheSize (the maximum cache size in MibiByte).
    // It is a subset of the data manager specification.
    virtual Record getProperties() const;

    // Modify data manager properties.
    // Only MaxCacheSize can be used. It is similar to function setCacheSize
    // with <src>canExceedNrBuckets=False</src>.
    virtual void setProperties (const Record& spec);

    // Set the flag to "data has changed since last flush".
    void setDataChanged();

    // Derive the tile shape from the hypercube shape for the given
    // number of pixels per tile. It is tried to get the same number
    // of tiles for each dimension.
    // When a weight vector is given, the number of tiles for a dimension
    // is proportional to the weight.
    // <br>After the initial guess it tries to optimize it by trying
    // to waste as little space as possible, while trying to keep as close
    // to the initial guess. The given tolerance (possibly per axis)
    // gives the minimum and maximum possible length of a tile axis
    // (minimum = initial_guess*tolerance; maximum = initial_guess/tolerance).
    // The heuristic is such that a tile axis length dividing the cube length
    // exactly is always favoured.
    // The test program <src>tTiledStMan</src> can be used to see how
    // the algorithm works out for a given tile size and cube shape.
    // <group>
    static IPosition makeTileShape (const IPosition& hypercubeShape,
                    Double tolerance = 0.5,
                    uInt64 maxNrPixelsPerTile = 4*1024*1024);
    static IPosition makeTileShape (const IPosition& hypercubeShape,
                    const Vector<double>& weight,
                    const Vector<double>& tolerance,
                    uInt64 maxNrPixelsPerTile = 4*1024*1024);
    // </group>

    // Set the maximum cache size (in MiB) in a non-persistent way.
    virtual void setMaximumCacheSize (uInt nMiB);

    // Get the current maximum cache size (in MiB (MibiByte)).
    uInt maximumCacheSize() const;

    // Get the current cache size (in buckets) for the hypercube in
    // the given row.
    uInt cacheSize (rownr_t rownr) const;

    // Get the hypercube shape of the data in the given row.
    const IPosition& hypercubeShape (rownr_t rownr) const;

    // Get the tile shape of the data in the given row.
    const IPosition& tileShape (rownr_t rownr) const;

    // Get the bucket size (in bytes) of the hypercube in the given row.
    uInt64 bucketSize (rownr_t rownr) const;

    // Can the tiled storage manager handle changing array shapes?
    // The default is no (but TiledCellStMan can).
    virtual Bool canChangeShape() const;

    // Can the tiled storage manager access an entire column.
    // TiledColumnStMan can always do that.
    // The others might be able to do it (for this time).
    // The default implementation returns True if there is only 1 hypercube.
    virtual Bool canAccessColumn() const;

    // The data manager supports use of MultiFile.
    virtual Bool hasMultiFileSupport() const;

    // Calculate the cache size (in buckets) for accessing the hypercube
    // containing the given row. It takes the maximum cache size into
    // account (allowing an overdraft of 10%).
    // It uses the given axisPath (i.e. traversal order) to determine
    // the optimum size. A window can be specified to indicate that only
    // the given subset of the hypercube will be accessed.
    // <br>
    // The length of the slice and window arguments and <src>axisPath</src>
    // must be less or equal to the dimensionality of the hypercube.
    // The non-specified <src>windowStart</src> parts default to 0.
    // The non-specified <src>windowLength</src> parts default to
    // the hypercube shape.
    // The non-specified <src>sliceShape</src> parts default to 1.
    // <br>
    // Axispath = [2,0,1] indicates that the z-axis changes most rapidly,
    // thereafter x and y. An axis can occur only once in the axisPath.
    // The non-specified <src>axisPath</src> parts get the natural order.
    // E.g. in the previous example axisPath=[2] defines the same path.
    // <br>When forceSmaller is False, the cache is not resized when the
    // new size is smaller.
    // <br>A flag is set indicating that the TSMDataColumn
    // access functions do not need to size the cache.
    uInt calcCacheSize (rownr_t rownr, const IPosition& sliceShape,
            const IPosition& windowStart,
            const IPosition& windowLength,
            const IPosition& axisPath) const;

    // Set the cache size using the <src>calcCacheSize</src>
    // function mentioned above.
    void setCacheSize (rownr_t rownr, const IPosition& sliceShape,
               const IPosition& windowStart,
               const IPosition& windowLength,
               const IPosition& axisPath,
               Bool forceSmaller);

    // Set the cache size for accessing the hypercube containing the given row.
    // When the give cache size exceeds the maximum cache size with more
    // than 10%, the maximum cache size is used instead.
    // <br>When forceSmaller is False, the cache is not resized when the
    // new size is smaller.
    // <br>A flag is set indicating that the TSMDataColumn
    // access functions do not need to size the cache.
    void setCacheSize (rownr_t rownr, uInt nbuckets, Bool forceSmaller);

    // Sets the cache size using the hypercube instead of the row number.
    // Useful for iterating over all hypercubes.
    void setHypercubeCacheSize (uInt hypercube, uInt nbuckets, Bool forceSmaller);

    // Determine if the user set the cache size (using setCacheSize).
    Bool userSetCache (rownr_t rownr) const;

    // Empty the caches used by the hypercubes in this storage manager.
    // It will flush the caches as needed and remove all buckets from them
    // resulting in a possibly large drop in memory used.
    // It also clears the userSetCache flag.
    void emptyCaches();

    // Show the statistics of all caches used.
    void showCacheStatistics (ostream& os) const;

    // Get the length of the data for the given number of pixels.
    // This can be used to calculate the length of a tile.
    uInt64 getLengthOffset (uInt64 nrPixels, Block<uInt>& dataOffset,
                            Block<uInt>& localOffset,
                            uInt& localTileLength) const;

    // Get the number of coordinate vectors.
    uInt nrCoordVector() const;

    // Get the nr of rows in this storage manager.
    rownr_t nrow() const;

    // Does the storage manager allow to add rows? (yes)
    Bool canAddRow() const;

    // Get the default tile shape.
    // By default it returns a zero-length IPosition.
    virtual IPosition defaultTileShape() const;

    // Return the number of hypercubes.
    uInt nhypercubes() const;

    // Test if only one hypercube is used by this storage manager.
    // If not, throw an exception. Otherwise return the hypercube.
    virtual TSMCube* singleHypercube();

    // Get the given hypercube.
    // <group>
    const TSMCube* getTSMCube (uInt hypercube) const;
    TSMCube* getTSMCube (uInt hypercube);
    // </group>
    
    // Get the hypercube in which the given row is stored.
    // <group>
    const TSMCube* getHypercube (rownr_t rownr) const;
    virtual TSMCube* getHypercube (rownr_t rownr) = 0;
    // </group>

    // Get the hypercube in which the given row is stored.
    // It also returns the position of the row in that hypercube.
    virtual TSMCube* getHypercube (rownr_t rownr, IPosition& position) = 0;

    // Make the correct TSMCube type (depending on tsmOption()).
    TSMCube* makeTSMCube (TSMFile* file, const IPosition& cubeShape,
                          const IPosition& tileShape,
                          const Record& values, Int64 fileOffset=-1);

    // Read a tile and convert the data to local format.
    void readTile (char* local, const Block<uInt>& localOffset,
           const char* external, const Block<uInt>& externalOffset,
           uInt nrpixels);

    // Write a tile after converting the data to external format.
    void writeTile (char* external, const Block<uInt>& externalOffset,
            const char* local, const Block<uInt>& localOffset,
            uInt nrpixels);

    // Get the TSMFile object with the given sequence number.
    TSMFile* getFile (uInt sequenceNumber);

    // Open the storage manager for an existing table.
    virtual rownr_t open64 (rownr_t nrrow, AipsIO&);

    // Resync the storage manager with the new file contents.
    virtual rownr_t resync64 (rownr_t nrrow);

    // Reopen all files used in this storage manager for read/write access.
    virtual void reopenRW();

    // The data manager will be deleted (because all its columns are
    // requested to be deleted).
    // So clean up the things needed (e.g. delete files).
    virtual void deleteManager();

    // Create a column in the storage manager on behalf of a table column.
    // <group>
    // Create a scalar column.
    DataManagerColumn* makeScalarColumn (const String& name, int dataType,
                     const String& dataTypeID);
    // Create a direct array column.
    DataManagerColumn* makeDirArrColumn (const String& name, int dataType,
                     const String& dataTypeID);
    // Create an indirect array column.
    DataManagerColumn* makeIndArrColumn (const String& name, int dataType,
                     const String& dataTypeID);
    // </group>

    // The TiledStMan wants to do reallocateColumn.
    Bool canReallocateColumns() const;

    // Reallocate the column object if it is part of this data manager.
    // It returns a pointer to the new column object.
    // It is used to remove the indirection of the TSMColumn objects
    // resulting in only one iso. two virtual column calls to get the data.
    DataManagerColumn* reallocateColumn (DataManagerColumn* column);

    // Set the shape and tile shape of a hypercube.
    // By default it throws an "impossible" exception.
    virtual void setShape (rownr_t rownr, TSMCube* hypercube,
               const IPosition& shape,
               const IPosition& tileShape);

    // Check the shape to be set for a hypercube.
    // It checks if it matches predefined (fixed shape) columns
    // and the shape of already defined coordinate columns.
    void checkCubeShape (const TSMCube* hypercube,
             const IPosition& cubeShape) const;

    // Get the data type of the coordinate column with the given name.
    // An exception is thrown when the column is unknown.
    int coordinateDataType (const String& columnName) const;

    // Initialize the new coordinates for the given cube.
    void initCoordinates (TSMCube* hypercube);

    // Get pointer to data column object.
    const TSMDataColumn* getDataColumn (uInt colnr) const
      { return dataCols_p[colnr]; }

protected:
    // Set the persistent maximum cache size (in MiB).
    void setPersMaxCacheSize (uInt nMiB);

    // Get the bindings of the columns with the given names.
    // If bound, the pointer to the TSMColumn object is stored in the block.
    // If mustExist is True, an exception is thrown if the column
    // is not bound.
    // It returns the number of bound columns.
    uInt getBindings (const Vector<String>& columnNames,
              PtrBlock<TSMColumn*>& colSet,
              Bool mustExist) const;

    // Function setup calls this function to allow the derived class
    // to check specific information. In case of errors, an exception
    // should be thrown.
    // By default it does nothing.
    virtual void setupCheck (const TableDesc& tableDesc,
                 const Vector<String>& dataNames) const;

    // Get the table description needed for the hypercolumn description.
    virtual const TableDesc& getDesc() const;

    // Check if values are given in the record for all columns in
    // the block. Also check if the data types are correct.
    // An exception is thrown if something is incorrect.
    void checkValues (const PtrBlock<TSMColumn*>& colSet,
              const Record& values) const;

    // Check if the coordinate values are correct.
    // This calls checkValues and checks if their shapes match the
    // hypercube shape.
    // An exception is thrown if invalid.
    void checkCoordinates (const PtrBlock<TSMColumn*>& coordColSet,
               const IPosition& cubeShape,
               const Record& values) const;

    // Check if the shapes of FixedShape data and coordinate columns match.
    // An exception is thrown if not.
    void checkShapeColumn (const IPosition& shape) const;

    // Check if the cube shape matches that of defined coordinates.
    void checkCoordinatesShapes (const TSMCube* hypercube,
                 const IPosition& cubeShape) const;

    // Check if the hypercube to be added is correctly defined.
    void checkAddHypercube (const IPosition& cubeShape,
                const Record& values) const;

    // Make a new TSMCube object.
    TSMCube* makeHypercube (const IPosition& cubeShape,
                const IPosition& tileShape,
                const Record& values);

    // Get the index of the hypercube with the given id-values.
    // If not found, -1 is returned.
    Int getCubeIndex (const Record& idValues) const;
    
    // Determine how many rows need to be added for an extension
    // (in the last dimension) of a hypercube with the given shape.
    rownr_t addedNrrow (const IPosition& shape, uInt incrInLastDim) const;

    // Flush the caches of all hypercubes.
    // If data have put and fsync is set, fsync all files.
    Bool flushCaches (Bool fsync);

    // Let a derived class read the header info.
    // This is used by the open and resync function.
    virtual void readHeader (rownr_t nrrow, Bool firstTime) = 0;

    // Create the TSM header file.
    // It creates an AipsIO object for it.
    AipsIO* headerFileCreate();

    // Open the TSM header file.
    // It creates an AipsIO object for it.
    AipsIO* headerFileOpen();

    // Write the data into the header file.
    // The given number of TSMCube objects have to be written.
    void headerFilePut (AipsIO& headerFile, uInt64 nrCube);

    // Read the data from the header file.
    // When done for the first time, setup() is called to initialize
    // the various variables (using the extraNdim variable).
    // It returns the version of the AipsIO object in the header.
    uInt headerFileGet (AipsIO& headerFile, rownr_t tabNrrow, Bool firstTime,
            Int extraNdim);

    // Close the header file.
    // It deletes the AipsIO object.
    void headerFileClose (AipsIO* headerFile);

    // Set up the TiledStMan variables from the table description.
    // The argument specifies the number of extra dimensions for the
    // hypercube compared to the data array (usually 0 or 1).
    // It is only used if no hypercolumn definition exists.
    // -1 means that the hypercolumn definition has to be present.
    void setup (Int extraNdim=-1);

    // Create a TSMFile object and store its pointer at the given index
    // in the block.
    void createFile (uInt index);

    // Convert the scalar data type to an array data type.
    // This function is temporary and can disappear when the ColumnDesc
    // classes use type TpArray*.
    int arrayDataType (int dataType) const;


    //# Declare all data members.
    // The name of the hypercolumn.
    String  hypercolumnName_p;
    // The number of rows in the columns.
    rownr_t nrrow_p;
    // The assembly of all columns.
    PtrBlock<TSMColumn*>  colSet_p;
    // The assembly of all data columns.
    PtrBlock<TSMDataColumn*> dataCols_p;
    PtrBlock<TSMColumn*>  dataColSet_p;
    // The assembly of all id columns.
    PtrBlock<TSMColumn*>  idColSet_p;
    // The assembly of all coordinate columns.
    PtrBlock<TSMColumn*>  coordColSet_p;
    // The assembly of all TSMFile objects.
    // The first file is for all non-extensible cubes, while the others
    // are for one file per extensible cube.
    PtrBlock<TSMFile*> fileSet_p;
    // The assembly of all TSMCube objects.
    PtrBlock<TSMCube*> cubeSet_p;
    // The persistent maximum cache size (in MiB) for a hypercube.
    uInt      persMaxCacheSize_p;
    // The actual maximum cache size for a hypercube (in MiB).
    uInt      maxCacheSize_p;
    // The dimensionality of the hypercolumn.
    uInt      nrdim_p;
    // The number of vector coordinates.
    uInt      nrCoordVector_p;
    // The fixed cell shape.
    IPosition fixedCellShape_p;
    // Has any data changed since the last flush?
    Bool      dataChanged_p;

private:
    // Forbid copy constructor.
    TiledStMan (const TiledStMan&);

    // Forbid assignment.
    TiledStMan& operator= (const TiledStMan&);
};


inline uInt TiledStMan::maximumCacheSize() const
    { return maxCacheSize_p; }

inline uInt TiledStMan::nrCoordVector() const
    { return nrCoordVector_p; }

inline rownr_t TiledStMan::nrow() const
    { return nrrow_p; }

inline uInt TiledStMan::nhypercubes() const
    { return cubeSet_p.nelements(); }

inline void TiledStMan::setDataChanged()
    { dataChanged_p = True; }

inline const TSMCube* TiledStMan::getTSMCube (uInt hypercube) const
    { return const_cast<TiledStMan*>(this)->getTSMCube (hypercube); }

inline const TSMCube* TiledStMan::getHypercube (rownr_t rownr) const
    { return const_cast<TiledStMan*>(this)->getHypercube (rownr); }

inline void TiledStMan::setPersMaxCacheSize (uInt nMiB)
{
    persMaxCacheSize_p = nMiB;
    maxCacheSize_p = nMiB;
}




} //# NAMESPACE CASACORE - END

#endif