bes Updated for version 3.20.10
SuperChunk.h
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of the BES
4
5// Copyright (c) 2020 OPeNDAP, Inc.
6// Author: Nathan Potter<ndp@opendap.org>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23
24#ifndef HYRAX_GIT_SUPERCHUNK_H
25#define HYRAX_GIT_SUPERCHUNK_H
26
27
28#include <vector>
29#include <memory>
30#include <thread>
31#include <queue>
32#include <sstream>
33
34
35#include "Chunk.h"
36
37
38namespace dmrpp {
39
40// Forward Declaration;
41class DmrppArray;
42
48private:
49 std::string d_id;
50 DmrppArray *d_parent_array;
51 std::shared_ptr<http::url> d_data_url;
52 std::vector<std::shared_ptr<Chunk>> d_chunks;
53 unsigned long long d_offset;
54 unsigned long long d_size;
55 bool d_is_read;
56 char *d_read_buffer;
57
58 bool is_contiguous(std::shared_ptr<Chunk> candidate_chunk);
59 void map_chunks_to_buffer();
60 void read_aggregate_bytes();
61
62public:
63
64 explicit SuperChunk(const std::string sc_id, DmrppArray *parent=nullptr):
65 d_id(sc_id), d_parent_array(parent), d_data_url(nullptr), d_offset(0), d_size(0), d_is_read(false), d_read_buffer(nullptr){}
66
67 virtual ~SuperChunk(){
68 delete[] d_read_buffer;
69 }
70
71 virtual std::string id(){ return d_id; }
72
73 virtual bool add_chunk(std::shared_ptr<Chunk> candidate_chunk);
74
75 std::shared_ptr<http::url> get_data_url(){ return d_data_url; }
76 virtual unsigned long long get_size(){ return d_size; }
77 virtual unsigned long long get_offset(){ return d_offset; }
78
79 virtual void read(){
82 }
83 virtual void read_unconstrained(){
86 }
87
88 virtual void retrieve_data();
89 virtual void process_child_chunks();
91
92
93 virtual bool empty(){ return d_chunks.empty(); }
94
95 std::vector<std::shared_ptr<Chunk>> get_chunks(){ return d_chunks; }
96
97 std::string to_string(bool verbose) const;
98 virtual void dump(std::ostream & strm) const;
99};
100
106 std::thread::id parent_thread_id;
107 std::string parent_super_chunk_id;
108 std::shared_ptr<Chunk> chunk;
109 DmrppArray *array;
110 const vector<unsigned long long> &array_shape;
111
112 one_chunk_args(const string sc_id, std::shared_ptr<Chunk> c, DmrppArray *a, const vector<unsigned long long> &a_s)
113 : parent_thread_id(std::this_thread::get_id()), parent_super_chunk_id(sc_id), chunk(std::move(c)), array(a), array_shape(a_s) {}
114};
115
122 std::thread::id parent_thread_id;
123 std::string parent_super_chunk_id;
124 std::shared_ptr<Chunk> chunk;
125 DmrppArray *array;
126 const vector<unsigned long long> &array_shape;
127 const vector<unsigned long long> &chunk_shape;
128
129 one_chunk_unconstrained_args(const string sc_id, std::shared_ptr<Chunk> c, DmrppArray *a, const vector<unsigned long long> &a_s,
130 const vector<unsigned long long> &c_s)
131 : parent_thread_id(std::this_thread::get_id()), parent_super_chunk_id(sc_id), chunk(std::move(c)),
132 array(a), array_shape(a_s), chunk_shape(c_s) {}
133};
134
135void process_chunks_concurrent(
136 const string &super_chunk_id,
137 std::queue<shared_ptr<Chunk>> &chunks,
138 DmrppArray *array,
139 const std::vector<unsigned long long> &shape );
140
141void process_chunks_unconstrained_concurrent(
142 const string &super_chunk_id,
143 std::queue<std::shared_ptr<Chunk>> &chunks,
144 const std::vector<unsigned long long> &chunk_shape,
145 DmrppArray *array,
146 const std::vector<unsigned long long> &array_shape);
147
148
149
150}// namespace dmrpp
151
152
153#endif //HYRAX_GIT_SUPERCHUNK_H
Extend libdap::Array so that a handler can read data using a DMR++ file.
Definition: DmrppArray.h:68
A SuperChunk is a collection of contiguous Chunk objects along with optimized methods for data retrie...
Definition: SuperChunk.h:47
virtual void retrieve_data()
Cause the SuperChunk and all of it's subordinate Chunks to be read.
Definition: SuperChunk.cc:503
virtual bool add_chunk(std::shared_ptr< Chunk > candidate_chunk)
Attempts to add a new Chunk to this SuperChunk.
Definition: SuperChunk.cc:400
std::string to_string(bool verbose) const
Makes a string representation of the SuperChunk.
Definition: SuperChunk.cc:619
virtual void dump(std::ostream &strm) const
Writes the to_string() output to the stream strm.
Definition: SuperChunk.cc:640
virtual void process_child_chunks()
Reads the SuperChunk, inflates/deshuffles the subordinate chunks as required and copies the values in...
Definition: SuperChunk.cc:540
virtual void process_child_chunks_unconstrained()
Reads the SuperChunk, inflates/deshuffles the subordinate chunks as required and copies the values in...
Definition: SuperChunk.cc:578
Single argument structure for a thread that will process a single Chunk for a constrained array....
Definition: SuperChunk.h:105
Single argument structure for a thread that will process a single Chunk for an unconstrained array....
Definition: SuperChunk.h:121