1/********************************************************************
2 * Copyright (c) 2013 - 2014, Pivotal Inc.
3 * All rights reserved.
4 *
5 * Author: Zhanwei Wang
6 ********************************************************************/
7/********************************************************************
8 * 2014 -
9 * open source under Apache License Version 2.0
10 ********************************************************************/
11/**
12 * Licensed to the Apache Software Foundation (ASF) under one
13 * or more contributor license agreements. See the NOTICE file
14 * distributed with this work for additional information
15 * regarding copyright ownership. The ASF licenses this file
16 * to you under the Apache License, Version 2.0 (the
17 * "License"); you may not use this file except in compliance
18 * with the License. You may obtain a copy of the License at
19 *
20 * http://www.apache.org/licenses/LICENSE-2.0
21 *
22 * Unless required by applicable law or agreed to in writing, software
23 * distributed under the License is distributed on an "AS IS" BASIS,
24 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25 * See the License for the specific language governing permissions and
26 * limitations under the License.
27 */
28#ifndef _HDFS_LIBHDFS3_CLIENT_FILESYSTEMINTER_H_
29#define _HDFS_LIBHDFS3_CLIENT_FILESYSTEMINTER_H_
30
31#include <string>
32#include <vector>
33
34#include "BlockLocation.h"
35#include "DirectoryIterator.h"
36#include "FileStatus.h"
37#include "FileSystemKey.h"
38#include "FileSystemStats.h"
39#include "PeerCache.h"
40#include "Permission.h"
41#include "server/LocatedBlocks.h"
42#include "SessionConfig.h"
43#include "Unordered.h"
44#include "UserInfo.h"
45#include "XmlConfig.h"
46
47namespace Hdfs {
48
49class FileSystem;
50
51namespace Internal {
52
53class InputStreamInter;
54class OutputStreamInter;
55class FileSystemInter;
56
57struct FileSystemWrapper {
58public:
59 FileSystemWrapper(shared_ptr<FileSystemInter> fs) :
60 filesystem(fs) {
61 }
62
63 shared_ptr<FileSystemInter> filesystem;
64};
65
66class FileSystemInter {
67public:
68 /**
69 * Destroy a FileSystemInter instance
70 */
71 virtual ~FileSystemInter() {
72 }
73
74 /**
75 * Format the path to a absolute canonicalized path.
76 * @param path target path to be hendled.
77 * @return return a absolute canonicalized path.
78 */
79 virtual const std::string getStandardPath(const char * path) = 0;
80
81 /**
82 * To get the client unique ID.
83 * @return return the client unique ID.
84 */
85 virtual const char * getClientName() = 0;
86
87 /**
88 * Connect to hdfs
89 */
90 virtual void connect() = 0;
91
92 /**
93 * disconnect from hdfs
94 */
95 virtual void disconnect() = 0;
96
97 /**
98 * To get default number of replication.
99 * @return the default number of replication.
100 */
101 virtual int getDefaultReplication() const = 0;
102
103 /**
104 * To get the default block size.
105 * @return the default block size.
106 */
107 virtual int64_t getDefaultBlockSize() const = 0;
108
109 /**
110 * To get the home directory.
111 * @return home directory.
112 */
113 virtual std::string getHomeDirectory() const = 0;
114
115 /**
116 * To delete a file or directory.
117 * @param path the path to be deleted.
118 * @param recursive if path is a directory, delete the contents recursively.
119 * @return return true if success.
120 */
121 virtual bool deletePath(const char * path, bool recursive) = 0;
122
123 /**
124 * To create a directory which given permission.
125 * @param path the directory path which is to be created.
126 * @param permission directory permission.
127 * @return return true if success.
128 */
129 virtual bool mkdir(const char * path, const Permission & permission) = 0;
130
131 /**
132 * To create a directory which given permission.
133 * If parent path does not exits, create it.
134 * @param path the directory path which is to be created.
135 * @param permission directory permission.
136 * @return return true if success.
137 */
138 virtual bool mkdirs(const char * path, const Permission & permission) = 0;
139
140 /**
141 * To get path information.
142 * @param path the path which information is to be returned.
143 * @return the path information.
144 */
145 virtual FileStatus getFileStatus(const char * path) = 0;
146
147 /**
148 * Return an array containing hostnames, offset and size of
149 * portions of the given file.
150 *
151 * This call is most helpful with DFS, where it returns
152 * hostnames of machines that contain the given file.
153 *
154 * The FileSystem will simply return an elt containing 'localhost'.
155 *
156 * @param path path is used to identify an FS since an FS could have
157 * another FS that it could be delegating the call to
158 * @param start offset into the given file
159 * @param len length for which to get locations for
160 */
161 virtual std::vector<BlockLocation> getFileBlockLocations(
162 const char * path, int64_t start, int64_t len) = 0;
163
164 /**
165 * list the contents of a directory.
166 * @param path the directory path.
167 * @return Return a iterator to visit all elements in this directory.
168 */
169 virtual DirectoryIterator listDirectory(const char * path,
170 bool needLocation) = 0;
171
172 /**
173 * list all the contents of a directory.
174 * @param path The directory path.
175 * @return Return a vector of file informations in the directory.
176 */
177 virtual std::vector<FileStatus> listAllDirectoryItems(const char * path,
178 bool needLocation) = 0;
179
180 /**
181 * To set the owner and the group of the path.
182 * username and groupname cannot be empty at the same time.
183 * @param path the path which owner of group is to be changed.
184 * @param username new user name.
185 * @param groupname new group.
186 */
187 virtual void setOwner(const char * path, const char * username,
188 const char * groupname) = 0;
189
190 /**
191 * To set the access time or modification time of a path.
192 * @param path the path which access time or modification time is to be changed.
193 * @param mtime new modification time.
194 * @param atime new access time.
195 */
196 virtual void setTimes(const char * path, int64_t mtime, int64_t atime) = 0;
197
198 /**
199 * To set the permission of a path.
200 * @param path the path which permission is to be changed.
201 * @param permission new permission.
202 */
203 virtual void setPermission(const char * path,
204 const Permission & permission) = 0;
205
206 /**
207 * To set the number of replication.
208 * @param path the path which number of replication is to be changed.
209 * @param replication new number of replication.
210 * @return return true if success.
211 */
212 virtual bool setReplication(const char * path, short replication) = 0;
213
214 /**
215 * To rename a path.
216 * @param src old path.
217 * @param dst new path.
218 * @return return true if success.
219 */
220 virtual bool rename(const char * src, const char * dst) = 0;
221
222 /**
223 * To set working directory.
224 * @param path new working directory.
225 */
226 virtual void setWorkingDirectory(const char * path) = 0;
227
228 /**
229 * To get working directory.
230 * @return working directory.
231 */
232 virtual std::string getWorkingDirectory() const = 0;
233
234 /**
235 * To test if the path exist.
236 * @param path the path which is to be tested.
237 * @return return true if the path exist.
238 */
239 virtual bool exist(const char * path) = 0;
240
241 /**
242 * To get the file system status.
243 * @return the file system status.
244 */
245 virtual FileSystemStats getFsStats() = 0;
246
247 /**
248 * Truncate the file in the indicated path to the indicated size.
249 * @param src The path we will find the file to be truncated.
250 * @param size the position we will truncate to.
251 * @throw IOException
252 */
253 virtual bool truncate(const char * src, int64_t size) /* throw AccessControlException,
254 FileNotFoundException, UnresolvedLinkException, HdfsIOException */ = 0;
255
256 /**
257 * Get a valid Delegation Token.
258 *
259 * @param renewer the designated renewer for the token
260 * @return Token<DelegationTokenIdentifier>
261 * @throws IOException
262 */
263 virtual std::string getDelegationToken(const char * renewer) = 0;
264
265 /**
266 * Get a valid Delegation Token using the default user as renewer.
267 *
268 * @return Token<DelegationTokenIdentifier>
269 * @throws IOException
270 */
271 virtual std::string getDelegationToken() = 0;
272
273 /**
274 * Renew an existing delegation token.
275 *
276 * @param token delegation token obtained earlier
277 * @return the new expiration time
278 * @throws IOException
279 */
280 virtual int64_t renewDelegationToken(const std::string & token) = 0;
281
282 /**
283 * Cancel an existing delegation token.
284 *
285 * @param token delegation token
286 * @throws IOException
287 */
288 virtual void cancelDelegationToken(const std::string & token) = 0;
289
290 /**
291 * Get locations of the blocks of the specified file within the specified range.
292 * DataNode locations for each block are sorted by
293 * the proximity to the client.
294 *
295 * The client will then have to contact
296 * one of the indicated DataNodes to obtain the actual data.
297 *
298 * @param src file name
299 * @param offset range start offset
300 * @param length range length
301 * @param lbs output the returned blocks
302 */
303 virtual void getBlockLocations(const std::string & src, int64_t offset,
304 int64_t length, LocatedBlocks & lbs) = 0;
305
306 /**
307 * Create a new file entry in the namespace.
308 *
309 * @param src path of the file being created.
310 * @param masked masked permission.
311 * @param flag indicates whether the file should be
312 * overwritten if it already exists or create if it does not exist or append.
313 * @param createParent create missing parent directory if true
314 * @param replication block replication factor.
315 * @param blockSize maximum block size.
316 */
317 virtual void create(const std::string & src, const Permission & masked,
318 int flag, bool createParent, short replication,
319 int64_t blockSize) = 0;
320
321 /**
322 * Append to the end of the file.
323 *
324 * @param src path of the file being created.
325 * @return return the last partial block if any
326 */
327 virtual std::pair<shared_ptr<LocatedBlock>, shared_ptr<FileStatus> > append(
328 const std::string& src) = 0;
329
330 /**
331 * The client can give up on a block by calling abandonBlock().
332 * The client can then either obtain a new block, or complete or abandon the file.
333 * Any partial writes to the block will be discarded.
334 *
335 * @param b the block to be abandoned.
336 * @param src the file which the block belongs to.
337 */
338 virtual void abandonBlock(const ExtendedBlock & b,
339 const std::string & srcr) = 0;
340
341 /**
342 * A client that wants to write an additional block to the
343 * indicated filename (which must currently be open for writing)
344 * should call addBlock().
345 *
346 * addBlock() allocates a new block and datanodes the block data
347 * should be replicated to.
348 *
349 * addBlock() also commits the previous block by reporting
350 * to the name-node the actual generation stamp and the length
351 * of the block that the client has transmitted to data-nodes.
352 *
353 * @param src the file being created
354 * @param previous previous block
355 * @param excludeNodes a list of nodes that should not be allocated for the current block.
356 * @return return the new block.
357 */
358 virtual shared_ptr<LocatedBlock> addBlock(const std::string & src,
359 const ExtendedBlock * previous,
360 const std::vector<DatanodeInfo> & excludeNodes) = 0;
361
362 /**
363 * Get a datanode for an existing pipeline.
364 *
365 * @param src the file being written
366 * @param blk the block being written
367 * @param existings the existing nodes in the pipeline
368 * @param excludes the excluded nodes
369 * @param numAdditionalNodes number of additional datanodes
370 * @return return a new block information which contains new datanode.
371 */
372 virtual shared_ptr<LocatedBlock> getAdditionalDatanode(
373 const std::string & src, const ExtendedBlock & blk,
374 const std::vector<DatanodeInfo> & existings,
375 const std::vector<std::string> & storageIDs,
376 const std::vector<DatanodeInfo> & excludes,
377 int numAdditionalNodes) = 0;
378
379 /**
380 * The client is done writing data to the given filename, and would
381 * like to complete it.
382 *
383 * The function returns whether the file has been closed successfully.
384 * If the function returns false, the caller should try again.
385 *
386 * close() also commits the last block of file by reporting
387 * to the name-node the actual generation stamp and the length
388 * of the block that the client has transmitted to data-nodes.
389 *
390 * A call to complete() will not return true until all the file's
391 * blocks have been replicated the minimum number of times. Thus,
392 * DataNode failures may cause a client to call complete() several
393 * times before succeeding.
394 *
395 * @param src the file being written.
396 * @param last last block to be committed.
397 * @return return false if the client should retry.
398 */
399 virtual bool complete(const std::string & src,
400 const ExtendedBlock * last) = 0;
401
402 /**
403 * The client wants to report corrupted blocks (blocks with specified
404 * locations on datanodes).
405 * @param blocks Array of located blocks to report
406 */
407 /*virtual void reportBadBlocks(const std::vector<LocatedBlock> & blocks) = 0;*/
408
409 /**
410 * Write all metadata for this file into persistent storage.
411 * The file must be currently open for writing.
412 * @param src The const std::string & representation of the path
413 */
414 virtual void fsync(const std::string & src) = 0;
415
416 /**
417 * Get a new generation stamp together with an access token for
418 * a block under construction
419 *
420 * This method is called only when a client needs to recover a failed
421 * pipeline or set up a pipeline for appending to a block.
422 *
423 * @param block a block
424 * @return return a located block with a new generation stamp and an access token
425 */
426 virtual shared_ptr<LocatedBlock> updateBlockForPipeline(
427 const ExtendedBlock & block) = 0;
428
429 /**
430 * Update a pipeline for a block under construction
431 *
432 * @param clientName the name of the client
433 * @param oldBlock the old block
434 * @param newBlock the new block containing new generation stamp and length
435 * @param newNodes datanodes in the pipeline
436 * @throw HdfsIOException if any error occurs
437 */
438 virtual void updatePipeline(const ExtendedBlock & oldBlock,
439 const ExtendedBlock & newBlock,
440 const std::vector<DatanodeInfo> & newNodes,
441 const std::vector<std::string> & storageIDs) = 0;
442
443 /**
444 * register the output stream in filespace when it is opened.
445 */
446 virtual void registerOpenedOutputStream() = 0;
447
448 /**
449 * unregister the output stream from filespace when it is closed.
450 */
451 virtual bool unregisterOpenedOutputStream() = 0;
452
453 /**
454 * Get the configuration used in filesystem.
455 * @return return the configuration instance.
456 */
457 virtual const SessionConfig & getConf() const = 0;
458
459 /**
460 * Get the user used in filesystem.
461 * @return return the user information.
462 */
463 virtual const UserInfo & getUserInfo() const = 0;
464
465 /**
466 * Get a partial listing of the indicated directory
467 *
468 * @param src the directory name
469 * @param startAfter the name to start listing after encoded in java UTF8
470 * @param needLocation if the FileStatus should contain block locations
471 * @param dl append the returned directories.
472 * @return return true if there are more items.
473 */
474 virtual bool getListing(const std::string & src, const std::string & startAfter,
475 bool needLocation, std::vector<FileStatus> & dl) = 0;
476
477 /**
478 * To renew the lease.
479 *
480 * @return return false if the filesystem no long needs to renew lease.
481 */
482 virtual bool renewLease() = 0;
483
484 /**
485 * Get the peer cache.
486 *
487 * @return return the peer cache.
488 */
489 virtual PeerCache& getPeerCache() = 0;
490};
491
492}
493}
494#endif /* _HDFS_LIBHDFS3_CLIENT_FILESYSTEMINTER_H_ */
495