casacore
Loading...
Searching...
No Matches
TSMCube.h
Go to the documentation of this file.
1//# TSMCube.h: Tiled hypercube in a table
2//# Copyright (C) 1995,1996,1997,1999,2000,2001,2002
3//# Associated Universities, Inc. Washington DC, USA.
4//#
5//# This library is free software; you can redistribute it and/or modify it
6//# under the terms of the GNU Library General Public License as published by
7//# the Free Software Foundation; either version 2 of the License, or (at your
8//# option) any later version.
9//#
10//# This library is distributed in the hope that it will be useful, but WITHOUT
11//# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12//# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13//# License for more details.
14//#
15//# You should have received a copy of the GNU Library General Public License
16//# along with this library; if not, write to the Free Software Foundation,
17//# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18//#
19//# Correspondence concerning AIPS++ should be addressed as follows:
20//# Internet email: aips2-request@nrao.edu.
21//# Postal address: AIPS++ Project Office
22//# National Radio Astronomy Observatory
23//# 520 Edgemont Road
24//# Charlottesville, VA 22903-2475 USA
25//#
26//# $Id: TSMCube.h 21521 2014-12-10 08:06:42Z gervandiepen $
27
28#ifndef TABLES_TSMCUBE_H
29#define TABLES_TSMCUBE_H
30
31
32//# Includes
33#include <casacore/casa/aips.h>
34#include <casacore/tables/DataMan/TSMShape.h>
35#include <casacore/casa/Containers/Record.h>
36#include <casacore/casa/Arrays/IPosition.h>
37#include <casacore/casa/OS/Conversion.h>
38#include <casacore/casa/iosfwd.h>
39
40namespace casacore { //# NAMESPACE CASACORE - BEGIN
41
42//# Forward declarations
43class TiledStMan;
44class TSMFile;
45class TSMColumn;
46class BucketCache;
47template<class T> class Block;
48
49// <summary>
50// Tiled hypercube in a table
51// </summary>
52
53// <use visibility=local>
54
55// <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
56// </reviewed>
57
58// <prerequisite>
59//# Classes you should understand before using this one.
60// <li> <linkto class=TiledStMan>TiledStMan</linkto>
61// <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
62// for a discussion of the maximum cache size
63// <li> <linkto class=TSMFile>TSMFile</linkto>
64// <li> <linkto class=BucketCache>BucketCache</linkto>
65// </prerequisite>
66
67// <etymology>
68// TSMCube represents a hypercube in the Tiled Storage Manager.
69// </etymology>
70
71// <synopsis>
72// TSMCube defines a tiled hypercube. The data is stored in a TSMFile
73// object and accessed using a BucketCache object. The hypercube can
74// be extensible in its last dimension to support tables with a size
75// which is not known in advance.
76// <br>
77// Normally hypercubes share the same TSMFile object, but extensible
78// hypercubes have their own TSMFile object (to be extensible).
79// If the hypercolumn has multiple data columns, their cells share the same
80// tiles. Per tile data column A appears first, thereafter B, etc..
81// <br>
82// The data in the cache is held in external format and is converted
83// when accessed. The alternative would be to hold it in the cache in
84// local format and convert it when read/written from the file. It was
85// felt that the latter approach would generate more needless conversions.
86// <p>
87// The possible id and coordinate values are stored in a Record
88// object. They are written in the main hypercube AipsIO file.
89// <p>
90// TSMCube uses the maximum cache size set for a Tiled Storage manager.
91// The description of class
92// <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
93// contains a discussion about the effect of setting the maximum cache size.
94// </synopsis>
95
96// <motivation>
97// TSMCube encapsulates all operations on a hypercube.
98// </motivation>
99
100//# <todo asof="$DATE:$">
101//# A List of bugs, limitations, extensions or planned refinements.
102//# </todo>
103
104
106{
107public:
108 // Define the possible access types for TSMDataColumn.
116
117 // Construct the hypercube using the given file with the given shape.
118 // The record contains the id and possible coordinate values.
119 // <br>If the cubeshape is empty, the hypercube is still undefined and
120 // can be added later with setShape. That is only used by TiledCellStMan.
121 // <br> The fileOffset argument is meant for class TiledFileAccess.
122 TSMCube (TiledStMan* stman, TSMFile* file,
123 const IPosition& cubeShape,
124 const IPosition& tileShape,
125 const Record& values,
126 Int64 fileOffset,
127 Bool useDerived = False);
128
129 // Reconstruct the hypercube by reading its data from the AipsIO stream.
130 // It will link itself to the correct TSMFile. The TSMFile objects
131 // must have been reconstructed in advance.
132 TSMCube (TiledStMan* stman, AipsIO& ios,
133 Bool useDerived = False);
134
135 virtual ~TSMCube();
136
137 // Flush the data in the cache.
138 virtual void flushCache();
139
140 // Clear the cache, so data will be reread.
141 // If wanted, the data is flushed before the cache is cleared.
142 void clearCache (Bool doFlush = True);
143
144 // Empty the cache.
145 // It will flush the cache as needed and remove all buckets from it
146 // resulting in a possibly large drop in memory used.
147 // It'll also clear the <src>userSetCache_p</src> flag.
149
150 // Show the cache statistics.
151 virtual void showCacheStatistics (ostream& os) const;
152
153 // Put the data of the object into the AipsIO stream.
154 void putObject (AipsIO& ios);
155
156 // Get the data of the object from the AipsIO stream.
157 // It returns the data manager sequence number, which is -1 if
158 // no file is attached to the cube (for cells without a value).
160
161 // Resync the object with the data file.
162 // It reads the object, and adjusts the cache.
163 virtual void resync (AipsIO& ios);
164
165 // Is the hypercube extensible?
167
168 // Get the bucket size (bytes).
169 // It is the length of a tile in external format.
170 uInt bucketSize() const;
171
172 // Get the length of a tile (in bytes) in local format.
173 uInt localTileLength() const;
174
175 // Set the hypercube shape.
176 // This is only possible if the shape was not defined yet.
177 virtual void setShape (const IPosition& cubeShape,
178 const IPosition& tileShape);
179
180 // Get the shape of the hypercube.
181 const IPosition& cubeShape() const;
182
183 // Get the shape of the tiles.
184 const IPosition& tileShape() const;
185
186 // Get the shape of the data cells in the cube.
188
189 // Get the size of a coordinate (i.e. the number of values in it).
190 // If not defined, it returns zero.
191 uInt coordinateSize (const String& coordinateName) const;
192
193 // Get the record containing the id and coordinate values.
194 // It is used by TSMIdColumn and TSMCoordColumn.
195 // <group>
196 const Record& valueRecord() const;
198 // </group>
199
200 // Test if the id values match.
202 const Record& idValues);
203
204 // Extend the last dimension of the cube with the given number.
205 // The record can contain the coordinates of the elements added.
206 virtual void extend (uInt64 nr, const Record& coordValues,
207 const TSMColumn* lastCoordColumn);
208
209 // Extend the coordinates vector for the given coordinate
210 // to the given length with the given coordValues.
211 // It will be initialized to zero if no coordValues are given.
212 // If the coordinate vector does not exist yet, it will be created.
213 void extendCoordinates (const Record& coordValues,
214 const String& coordName, uInt length);
215
216 // Read or write a section in the cube.
217 // It is assumed that the section buffer is long enough.
218 virtual void accessSection (const IPosition& start, const IPosition& end,
219 char* section, uInt colnr,
220 uInt localPixelSize, uInt externalPixelSize,
221 Bool writeFlag);
222
223 // Read or write a section in a strided way.
224 // It is assumed that the section buffer is long enough.
225 virtual void accessStrided (const IPosition& start, const IPosition& end,
226 const IPosition& stride,
227 char* section, uInt colnr,
228 uInt localPixelSize, uInt externalPixelSize,
229 Bool writeFlag);
230
231 // Get the current cache size (in buckets).
233
234 // Calculate the cache size (in buckets) for the given slice
235 // and access path.
236 // <group>
237 uInt calcCacheSize (const IPosition& sliceShape,
238 const IPosition& windowStart,
239 const IPosition& windowLength,
240 const IPosition& axisPath) const;
242 const IPosition& tileShape,
243 Bool extensible,
244 const IPosition& sliceShape,
245 const IPosition& windowStart,
246 const IPosition& windowLength,
247 const IPosition& axisPath,
248 uInt maxCacheSizeMiB, uInt bucketSize);
249 // </group>
250
251 // Set the cache size for the given slice and access path.
252 virtual void setCacheSize (const IPosition& sliceShape,
253 const IPosition& windowStart,
254 const IPosition& windowLength,
255 const IPosition& axisPath,
256 Bool forceSmaller, Bool userSet);
257
258 // Resize the cache object.
259 // If forceSmaller is False, the cache will only be resized when it grows.
260 // If the given size exceeds the maximum size with more
261 // than 10%, the maximum size will be used.
262 // The cacheSize has to be given in buckets.
263 // <br>The flag <src>userSet</src> inidicates if the cache size is set by
264 // the user (by an Accessor object) or automatically (by TSMDataColumn).
265 virtual void setCacheSize (uInt cacheSize, Bool forceSmaller, Bool userSet);
266
267 // Validate the cache size (in buckets).
268 // This means it will return the given cache size (in buckets) if
269 // smaller than the maximum cache size (given in MiB).
270 // Otherwise the maximum is returned.
271 // <group>
275 // </group>
276
277 // Determine if the user set the cache size (using setCacheSize).
278 Bool userSetCache() const;
279
280 // Functions for TSMDataColumn to keep track of the last type of
281 // access to a hypercube. It uses it to determine if the cache
282 // has to be reset.
283 // <group>
285 const IPosition& getLastColSlice() const;
286 void setLastColAccess (AccessType type);
287 void setLastColSlice (const IPosition& slice);
288 // </group>
289
290protected:
291 // Initialize the various variables.
292 // <group>
293 void setup();
295 // </group>
296
297 // Adjust the tile shape to the hypercube shape.
298 // A size of 0 gets set to 1.
299 // A tile size > cube size gets set to the cube size.
301 const IPosition& tileShape) const;
302
303 // Resize the IPosition member variables used in accessSection()
304 // if nrdim_p changes value.
306
307private:
308 // Forbid copy constructor.
309 TSMCube (const TSMCube&);
310
311 // Forbid assignment.
313
314 // Get the cache object.
315 // This will construct the cache object if not present yet.
317
318 // Construct the cache object (if not constructed yet).
319 virtual void makeCache();
320
321 // Resync the cache object.
322 virtual void resyncCache();
323
324 // Delete the cache object.
325 virtual void deleteCache();
326
327 // Access a line in a more optimized way.
328 void accessLine (char* section, uInt pixelOffset,
329 uInt localPixelSize,
330 Bool writeFlag, BucketCache* cachePtr,
331 const IPosition& startTile, uInt endTile,
332 const IPosition& startPixelInFirstTile,
333 uInt endPixelInLastTile,
334 uInt lineIndex);
335
336 // Define the callback functions for the BucketCache.
337 // <group>
338 static char* readCallBack (void* owner, const char* external);
339 static void writeCallBack (void* owner, char* external,
340 const char* local);
341 static char* initCallBack (void* owner);
342 static void deleteCallBack (void* owner, char* buffer);
343 // </group>
344
345 // Define the functions doing the actual read and write of the
346 // data in the tile and converting it to/from local format.
347 // <group>
348 char* readTile (const char* external);
349 void writeTile (char* external, const char* local);
350 // </group>
351
352protected:
353 //# Declare member variables.
354
355 char * cachedTile_p; // optimization to hold one tile chunk
356
357 // Pointer to the parent storage manager.
359 // Is the class used directly or only by a derived class only?
361 // The values of the possible id and coordinate columns.
363 // Is the hypercube extensible?
365 // Dimensionality of the hypercube.
367 // Number of tiles in the hypercube.
369 // The shape of the hypercube.
371 // The shape of the tiles in the hypercube.
373 // The number of tiles in each hypercube dimension.
375 // Precomputed tileShape information.
377 // Precomputed tilesPerDim information.
379 // Number of tiles in all but last dimension (used when extending).
381 // The tilesize in bytes.
383 // Pointer to the TSMFile object holding the data.
385 // Offset in the TSMFile object where the data of this hypercube starts.
387 // Offset for each data column in a tile (in external format).
389 // Offset for each data column in a tile (in local format).
391 // The bucket size in bytes (is equal to tile size in bytes).
393 // The tile size in bytes in local format.
395 // The bucket cache.
397 // Did the user set the cache size?
399 // Was the last column access to a cell, slice, or column?
401 // The slice shape of the last column access to a slice.
403
404 // IPosition variables used in accessSection(); declared here
405 // as member variables to avoid significant construction and
406 // desctruction overhead if they are local to accessSection()
407 // #tiles needed for the section
409 // First tile needed
411 // Last tile needed
413 // First pixel in first tile
415 // Last pixel in first tile
417 // Last pixel in last tile
419};
420
421
422
424{
425 if (cache_p == 0) {
426 makeCache();
427 }
428 return cache_p;
429}
431{
432 return bucketSize_p;
433}
435{
436 return localTileLength_p;
437}
438inline const IPosition& TSMCube::cubeShape() const
439{
440 return cubeShape_p;
441}
442inline const IPosition& TSMCube::tileShape() const
443{
444 return tileShape_p;
445}
446inline const Record& TSMCube::valueRecord() const
447{
448 return values_p;
449}
451{
452 return values_p;
453}
455{
456 return userSetCache_p;
457}
463{
464 return lastColSlice_p;
465}
467{
468 lastColAccess_p = type;
469}
470inline void TSMCube::setLastColSlice (const IPosition& slice)
471{
473 lastColSlice_p = slice;
474}
475
476
477
478
479} //# NAMESPACE CASACORE - END
480
481#endif
simple 1-D array
Definition Block.h:200
Cache for buckets in a part of a file.
void resize(size_t newSize, bool copy=true)
Old values are copied on resize if copy==true.
size_t nelements() const
The number of elements in this IPosition.
Definition IPosition.h:568
A drop-in replacement for Block<T*>.
Definition Block.h:814
String: the storage and methods of handling collections of characters.
Definition String.h:225
virtual ~TSMCube()
void extendCoordinates(const Record &coordValues, const String &coordName, uInt length)
Extend the coordinates vector for the given coordinate to the given length with the given coordValues...
uInt cacheSize() const
Get the current cache size (in buckets).
TSMFile * filePtr_p
Pointer to the TSMFile object holding the data.
Definition TSMCube.h:384
void emptyCache()
Empty the cache.
const Record & valueRecord() const
Get the record containing the id and coordinate values.
Definition TSMCube.h:446
void putObject(AipsIO &ios)
Put the data of the object into the AipsIO stream.
virtual void showCacheStatistics(ostream &os) const
Show the cache statistics.
IPosition lastColSlice_p
The slice shape of the last column access to a slice.
Definition TSMCube.h:402
uInt validateCacheSize(uInt cacheSize) const
Validate the cache size (in buckets).
virtual void setCacheSize(const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, Bool forceSmaller, Bool userSet)
Set the cache size for the given slice and access path.
virtual void resyncCache()
Resync the cache object.
IPosition tilesPerDim_p
The number of tiles in each hypercube dimension.
Definition TSMCube.h:374
void resizeTileSections()
Resize the IPosition member variables used in accessSection() if nrdim_p changes value.
virtual void makeCache()
Construct the cache object (if not constructed yet).
static void deleteCallBack(void *owner, char *buffer)
IPosition tileShape_p
The shape of the tiles in the hypercube.
Definition TSMCube.h:372
IPosition endPixelInFirstTile_p
Last pixel in first tile.
Definition TSMCube.h:416
Bool userSetCache() const
Determine if the user set the cache size (using setCacheSize).
Definition TSMCube.h:454
Block< uInt > localOffset_p
Offset for each data column in a tile (in local format).
Definition TSMCube.h:390
TSMCube(TiledStMan *stman, AipsIO &ios, Bool useDerived=False)
Reconstruct the hypercube by reading its data from the AipsIO stream.
char * readTile(const char *external)
Define the functions doing the actual read and write of the data in the tile and converting it to/fro...
uInt localTileLength() const
Get the length of a tile (in bytes) in local format.
Definition TSMCube.h:434
void setLastColAccess(AccessType type)
Definition TSMCube.h:466
virtual void resync(AipsIO &ios)
Resync the object with the data file.
const IPosition & tileShape() const
Get the shape of the tiles.
Definition TSMCube.h:442
uInt tileSize_p
The tilesize in bytes.
Definition TSMCube.h:382
IPosition adjustTileShape(const IPosition &cubeShape, const IPosition &tileShape) const
Adjust the tile shape to the hypercube shape.
TSMCube(const TSMCube &)
Forbid copy constructor.
uInt coordinateSize(const String &coordinateName) const
Get the size of a coordinate (i.e.
virtual void setShape(const IPosition &cubeShape, const IPosition &tileShape)
Set the hypercube shape.
Block< uInt > externalOffset_p
Offset for each data column in a tile (in external format).
Definition TSMCube.h:388
virtual void extend(uInt64 nr, const Record &coordValues, const TSMColumn *lastCoordColumn)
Extend the last dimension of the cube with the given number.
void accessLine(char *section, uInt pixelOffset, uInt localPixelSize, Bool writeFlag, BucketCache *cachePtr, const IPosition &startTile, uInt endTile, const IPosition &startPixelInFirstTile, uInt endPixelInLastTile, uInt lineIndex)
Access a line in a more optimized way.
IPosition cubeShape_p
The shape of the hypercube.
Definition TSMCube.h:370
uInt nrTiles_p
Number of tiles in the hypercube.
Definition TSMCube.h:368
void clearCache(Bool doFlush=True)
Clear the cache, so data will be reread.
uInt nrTilesSubCube_p
Number of tiles in all but last dimension (used when extending).
Definition TSMCube.h:380
static uInt calcCacheSize(const IPosition &cubeShape, const IPosition &tileShape, Bool extensible, const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, uInt maxCacheSizeMiB, uInt bucketSize)
TSMCube & operator=(const TSMCube &)
Forbid assignment.
TSMShape expandedTilesPerDim_p
Precomputed tilesPerDim information.
Definition TSMCube.h:378
IPosition startPixelInFirstTile_p
First pixel in first tile.
Definition TSMCube.h:414
void setLastColSlice(const IPosition &slice)
Definition TSMCube.h:470
TSMCube(TiledStMan *stman, TSMFile *file, const IPosition &cubeShape, const IPosition &tileShape, const Record &values, Int64 fileOffset, Bool useDerived=False)
Construct the hypercube using the given file with the given shape.
virtual void accessStrided(const IPosition &start, const IPosition &end, const IPosition &stride, char *section, uInt colnr, uInt localPixelSize, uInt externalPixelSize, Bool writeFlag)
Read or write a section in a strided way.
AccessType lastColAccess_p
Was the last column access to a cell, slice, or column?
Definition TSMCube.h:400
static char * readCallBack(void *owner, const char *external)
Define the callback functions for the BucketCache.
IPosition endPixelInLastTile_p
Last pixel in last tile.
Definition TSMCube.h:418
Bool matches(const PtrBlock< TSMColumn * > &idColSet, const Record &idValues)
Test if the id values match.
Bool useDerived_p
Is the class used directly or only by a derived class only?
Definition TSMCube.h:360
BucketCache * getCache()
Get the cache object.
Definition TSMCube.h:423
uInt nrdim_p
Dimensionality of the hypercube.
Definition TSMCube.h:366
Int64 fileOffset_p
Offset in the TSMFile object where the data of this hypercube starts.
Definition TSMCube.h:386
IPosition endTile_p
Last tile needed.
Definition TSMCube.h:412
const IPosition & cubeShape() const
Get the shape of the hypercube.
Definition TSMCube.h:438
Bool extensible_p
Is the hypercube extensible?
Definition TSMCube.h:364
virtual void flushCache()
Flush the data in the cache.
static uInt validateCacheSize(uInt cacheSize, uInt maxSizeMiB, uInt bucketSize)
void writeTile(char *external, const char *local)
uInt bucketSize_p
The bucket size in bytes (is equal to tile size in bytes).
Definition TSMCube.h:392
Record values_p
The values of the possible id and coordinate columns.
Definition TSMCube.h:362
Int getObject(AipsIO &ios)
Get the data of the object from the AipsIO stream.
void setup()
Initialize the various variables.
uInt calcCacheSize(const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath) const
Calculate the cache size (in buckets) for the given slice and access path.
virtual void deleteCache()
Delete the cache object.
TSMShape expandedTileShape_p
Precomputed tileShape information.
Definition TSMCube.h:376
static char * initCallBack(void *owner)
IPosition cellShape() const
Get the shape of the data cells in the cube.
static void writeCallBack(void *owner, char *external, const char *local)
AccessType
Define the possible access types for TSMDataColumn.
Definition TSMCube.h:109
TiledStMan * stmanPtr_p
Pointer to the parent storage manager.
Definition TSMCube.h:358
Bool userSetCache_p
Did the user set the cache size?
Definition TSMCube.h:398
virtual void accessSection(const IPosition &start, const IPosition &end, char *section, uInt colnr, uInt localPixelSize, uInt externalPixelSize, Bool writeFlag)
Read or write a section in the cube.
Record & rwValueRecord()
Definition TSMCube.h:450
IPosition startTile_p
First tile needed.
Definition TSMCube.h:410
BucketCache * cache_p
The bucket cache.
Definition TSMCube.h:396
AccessType getLastColAccess() const
Functions for TSMDataColumn to keep track of the last type of access to a hypercube.
Definition TSMCube.h:458
virtual void setCacheSize(uInt cacheSize, Bool forceSmaller, Bool userSet)
Resize the cache object.
Bool isExtensible() const
Is the hypercube extensible?
uInt localTileLength_p
The tile size in bytes in local format.
Definition TSMCube.h:394
const IPosition & getLastColSlice() const
Definition TSMCube.h:462
char * cachedTile_p
Definition TSMCube.h:355
IPosition nrTileSection_p
IPosition variables used in accessSection(); declared here as member variables to avoid significant c...
Definition TSMCube.h:408
uInt bucketSize() const
Get the bucket size (bytes).
Definition TSMCube.h:430
this file contains all the compiler specific defines
Definition mainpage.dox:28
const Bool False
Definition aipstype.h:44
unsigned int uInt
Definition aipstype.h:51
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition aipsxtype.h:38
LatticeExprNode length(const LatticeExprNode &expr, const LatticeExprNode &axis)
2-argument function to get the length of an axis.
int Int
Definition aipstype.h:50
bool Bool
Define the standard types used by Casacore.
Definition aipstype.h:42
const Bool True
Definition aipstype.h:43
unsigned long long uInt64
Definition aipsxtype.h:39