1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """
20 L{SFTPFile}
21 """
22
23 from binascii import hexlify
24 from collections import deque
25 import socket
26 import threading
27 import time
28
29 from paramiko.common import *
30 from paramiko.sftp import *
31 from paramiko.file import BufferedFile
32 from paramiko.sftp_attr import SFTPAttributes
33
34
36 """
37 Proxy object for a file on the remote server, in client mode SFTP.
38
39 Instances of this class may be used as context managers in the same way
40 that built-in Python file objects are.
41 """
42
43
44
45 MAX_REQUEST_SIZE = 32768
46
47 - def __init__(self, sftp, handle, mode='r', bufsize=-1):
48 BufferedFile.__init__(self)
49 self.sftp = sftp
50 self.handle = handle
51 BufferedFile._set_mode(self, mode, bufsize)
52 self.pipelined = False
53 self._prefetching = False
54 self._prefetch_done = False
55 self._prefetch_data = {}
56 self._prefetch_reads = []
57 self._saved_exception = None
58 self._reqs = deque()
59
61 self._close(async=True)
62
64 self._close(async=False)
65
66 - def _close(self, async=False):
67
68
69
70
71
72
73
74 if self._closed:
75 return
76 self.sftp._log(DEBUG, 'close(%s)' % hexlify(self.handle))
77 if self.pipelined:
78 self.sftp._finish_responses(self)
79 BufferedFile.close(self)
80 try:
81 if async:
82
83 self.sftp._async_request(type(None), CMD_CLOSE, self.handle)
84 else:
85 self.sftp._request(CMD_CLOSE, self.handle)
86 except EOFError:
87
88 pass
89 except (IOError, socket.error):
90
91 pass
92
94 k = [i for i in self._prefetch_reads if i[0] <= offset]
95 if len(k) == 0:
96 return False
97 k.sort(lambda x, y: cmp(x[0], y[0]))
98 buf_offset, buf_size = k[-1]
99 if buf_offset + buf_size <= offset:
100
101 return False
102 if buf_offset + buf_size >= offset + size:
103
104 return True
105
106 return self._data_in_prefetch_requests(buf_offset + buf_size, offset + size - buf_offset - buf_size)
107
109 """
110 if a block of data is present in the prefetch buffers, at the given
111 offset, return the offset of the relevant prefetch buffer. otherwise,
112 return None. this guarantees nothing about the number of bytes
113 collected in the prefetch buffer so far.
114 """
115 k = [i for i in self._prefetch_data.keys() if i <= offset]
116 if len(k) == 0:
117 return None
118 index = max(k)
119 buf_offset = offset - index
120 if buf_offset >= len(self._prefetch_data[index]):
121
122 return None
123 return index
124
126 """
127 read data out of the prefetch buffer, if possible. if the data isn't
128 in the buffer, return None. otherwise, behaves like a normal read.
129 """
130
131 while True:
132 offset = self._data_in_prefetch_buffers(self._realpos)
133 if offset is not None:
134 break
135 if self._prefetch_done or self._closed:
136 break
137 self.sftp._read_response()
138 self._check_exception()
139 if offset is None:
140 self._prefetching = False
141 return None
142 prefetch = self._prefetch_data[offset]
143 del self._prefetch_data[offset]
144
145 buf_offset = self._realpos - offset
146 if buf_offset > 0:
147 self._prefetch_data[offset] = prefetch[:buf_offset]
148 prefetch = prefetch[buf_offset:]
149 if size < len(prefetch):
150 self._prefetch_data[self._realpos + size] = prefetch[size:]
151 prefetch = prefetch[:size]
152 return prefetch
153
155 size = min(size, self.MAX_REQUEST_SIZE)
156 if self._prefetching:
157 data = self._read_prefetch(size)
158 if data is not None:
159 return data
160 t, msg = self.sftp._request(CMD_READ, self.handle, long(self._realpos), int(size))
161 if t != CMD_DATA:
162 raise SFTPError('Expected data')
163 return msg.get_string()
164
166
167 chunk = min(len(data), self.MAX_REQUEST_SIZE)
168 self._reqs.append(self.sftp._async_request(type(None), CMD_WRITE, self.handle, long(self._realpos), str(data[:chunk])))
169 if not self.pipelined or (len(self._reqs) > 100 and self.sftp.sock.recv_ready()):
170 while len(self._reqs):
171 req = self._reqs.popleft()
172 t, msg = self.sftp._read_response(req)
173 if t != CMD_STATUS:
174 raise SFTPError('Expected status')
175
176 return chunk
177
179 """
180 Set a timeout on read/write operations on the underlying socket or
181 ssh L{Channel}.
182
183 @see: L{Channel.settimeout}
184 @param timeout: seconds to wait for a pending read/write operation
185 before raising C{socket.timeout}, or C{None} for no timeout
186 @type timeout: float
187 """
188 self.sftp.sock.settimeout(timeout)
189
191 """
192 Returns the timeout in seconds (as a float) associated with the socket
193 or ssh L{Channel} used for this file.
194
195 @see: L{Channel.gettimeout}
196 @rtype: float
197 """
198 return self.sftp.sock.gettimeout()
199
201 """
202 Set blocking or non-blocking mode on the underiying socket or ssh
203 L{Channel}.
204
205 @see: L{Channel.setblocking}
206 @param blocking: 0 to set non-blocking mode; non-0 to set blocking
207 mode.
208 @type blocking: int
209 """
210 self.sftp.sock.setblocking(blocking)
211
212 - def seek(self, offset, whence=0):
213 self.flush()
214 if whence == self.SEEK_SET:
215 self._realpos = self._pos = offset
216 elif whence == self.SEEK_CUR:
217 self._pos += offset
218 self._realpos = self._pos
219 else:
220 self._realpos = self._pos = self._get_size() + offset
221 self._rbuffer = ''
222
224 """
225 Retrieve information about this file from the remote system. This is
226 exactly like L{SFTP.stat}, except that it operates on an already-open
227 file.
228
229 @return: an object containing attributes about this file.
230 @rtype: SFTPAttributes
231 """
232 t, msg = self.sftp._request(CMD_FSTAT, self.handle)
233 if t != CMD_ATTRS:
234 raise SFTPError('Expected attributes')
235 return SFTPAttributes._from_msg(msg)
236
238 """
239 Change the mode (permissions) of this file. The permissions are
240 unix-style and identical to those used by python's C{os.chmod}
241 function.
242
243 @param mode: new permissions
244 @type mode: int
245 """
246 self.sftp._log(DEBUG, 'chmod(%s, %r)' % (hexlify(self.handle), mode))
247 attr = SFTPAttributes()
248 attr.st_mode = mode
249 self.sftp._request(CMD_FSETSTAT, self.handle, attr)
250
251 - def chown(self, uid, gid):
252 """
253 Change the owner (C{uid}) and group (C{gid}) of this file. As with
254 python's C{os.chown} function, you must pass both arguments, so if you
255 only want to change one, use L{stat} first to retrieve the current
256 owner and group.
257
258 @param uid: new owner's uid
259 @type uid: int
260 @param gid: new group id
261 @type gid: int
262 """
263 self.sftp._log(DEBUG, 'chown(%s, %r, %r)' % (hexlify(self.handle), uid, gid))
264 attr = SFTPAttributes()
265 attr.st_uid, attr.st_gid = uid, gid
266 self.sftp._request(CMD_FSETSTAT, self.handle, attr)
267
269 """
270 Set the access and modified times of this file. If
271 C{times} is C{None}, then the file's access and modified times are set
272 to the current time. Otherwise, C{times} must be a 2-tuple of numbers,
273 of the form C{(atime, mtime)}, which is used to set the access and
274 modified times, respectively. This bizarre API is mimicked from python
275 for the sake of consistency -- I apologize.
276
277 @param times: C{None} or a tuple of (access time, modified time) in
278 standard internet epoch time (seconds since 01 January 1970 GMT)
279 @type times: tuple(int)
280 """
281 if times is None:
282 times = (time.time(), time.time())
283 self.sftp._log(DEBUG, 'utime(%s, %r)' % (hexlify(self.handle), times))
284 attr = SFTPAttributes()
285 attr.st_atime, attr.st_mtime = times
286 self.sftp._request(CMD_FSETSTAT, self.handle, attr)
287
289 """
290 Change the size of this file. This usually extends
291 or shrinks the size of the file, just like the C{truncate()} method on
292 python file objects.
293
294 @param size: the new size of the file
295 @type size: int or long
296 """
297 self.sftp._log(DEBUG, 'truncate(%s, %r)' % (hexlify(self.handle), size))
298 attr = SFTPAttributes()
299 attr.st_size = size
300 self.sftp._request(CMD_FSETSTAT, self.handle, attr)
301
302 - def check(self, hash_algorithm, offset=0, length=0, block_size=0):
303 """
304 Ask the server for a hash of a section of this file. This can be used
305 to verify a successful upload or download, or for various rsync-like
306 operations.
307
308 The file is hashed from C{offset}, for C{length} bytes. If C{length}
309 is 0, the remainder of the file is hashed. Thus, if both C{offset}
310 and C{length} are zero, the entire file is hashed.
311
312 Normally, C{block_size} will be 0 (the default), and this method will
313 return a byte string representing the requested hash (for example, a
314 string of length 16 for MD5, or 20 for SHA-1). If a non-zero
315 C{block_size} is given, each chunk of the file (from C{offset} to
316 C{offset + length}) of C{block_size} bytes is computed as a separate
317 hash. The hash results are all concatenated and returned as a single
318 string.
319
320 For example, C{check('sha1', 0, 1024, 512)} will return a string of
321 length 40. The first 20 bytes will be the SHA-1 of the first 512 bytes
322 of the file, and the last 20 bytes will be the SHA-1 of the next 512
323 bytes.
324
325 @param hash_algorithm: the name of the hash algorithm to use (normally
326 C{"sha1"} or C{"md5"})
327 @type hash_algorithm: str
328 @param offset: offset into the file to begin hashing (0 means to start
329 from the beginning)
330 @type offset: int or long
331 @param length: number of bytes to hash (0 means continue to the end of
332 the file)
333 @type length: int or long
334 @param block_size: number of bytes to hash per result (must not be less
335 than 256; 0 means to compute only one hash of the entire segment)
336 @type block_size: int
337 @return: string of bytes representing the hash of each block,
338 concatenated together
339 @rtype: str
340
341 @note: Many (most?) servers don't support this extension yet.
342
343 @raise IOError: if the server doesn't support the "check-file"
344 extension, or possibly doesn't support the hash algorithm
345 requested
346
347 @since: 1.4
348 """
349 t, msg = self.sftp._request(CMD_EXTENDED, 'check-file', self.handle,
350 hash_algorithm, long(offset), long(length), block_size)
351 ext = msg.get_string()
352 alg = msg.get_string()
353 data = msg.get_remainder()
354 return data
355
357 """
358 Turn on/off the pipelining of write operations to this file. When
359 pipelining is on, paramiko won't wait for the server response after
360 each write operation. Instead, they're collected as they come in.
361 At the first non-write operation (including L{close}), all remaining
362 server responses are collected. This means that if there was an error
363 with one of your later writes, an exception might be thrown from
364 within L{close} instead of L{write}.
365
366 By default, files are I{not} pipelined.
367
368 @param pipelined: C{True} if pipelining should be turned on for this
369 file; C{False} otherwise
370 @type pipelined: bool
371
372 @since: 1.5
373 """
374 self.pipelined = pipelined
375
377 """
378 Pre-fetch the remaining contents of this file in anticipation of
379 future L{read} calls. If reading the entire file, pre-fetching can
380 dramatically improve the download speed by avoiding roundtrip latency.
381 The file's contents are incrementally buffered in a background thread.
382
383 The prefetched data is stored in a buffer until read via the L{read}
384 method. Once data has been read, it's removed from the buffer. The
385 data may be read in a random order (using L{seek}); chunks of the
386 buffer that haven't been read will continue to be buffered.
387
388 @since: 1.5.1
389 """
390 size = self.stat().st_size
391
392 chunks = []
393 n = self._realpos
394 while n < size:
395 chunk = min(self.MAX_REQUEST_SIZE, size - n)
396 chunks.append((n, chunk))
397 n += chunk
398 if len(chunks) > 0:
399 self._start_prefetch(chunks)
400
401 - def readv(self, chunks):
402 """
403 Read a set of blocks from the file by (offset, length). This is more
404 efficient than doing a series of L{seek} and L{read} calls, since the
405 prefetch machinery is used to retrieve all the requested blocks at
406 once.
407
408 @param chunks: a list of (offset, length) tuples indicating which
409 sections of the file to read
410 @type chunks: list(tuple(long, int))
411 @return: a list of blocks read, in the same order as in C{chunks}
412 @rtype: list(str)
413
414 @since: 1.5.4
415 """
416 self.sftp._log(DEBUG, 'readv(%s, %r)' % (hexlify(self.handle), chunks))
417
418 read_chunks = []
419 for offset, size in chunks:
420
421 if self._data_in_prefetch_buffers(offset) or self._data_in_prefetch_requests(offset, size):
422 continue
423
424
425 while size > 0:
426 chunk_size = min(size, self.MAX_REQUEST_SIZE)
427 read_chunks.append((offset, chunk_size))
428 offset += chunk_size
429 size -= chunk_size
430
431 self._start_prefetch(read_chunks)
432
433 for x in chunks:
434 self.seek(x[0])
435 yield self.read(x[1])
436
437
438
439
440
442 try:
443 return self.stat().st_size
444 except:
445 return 0
446
448 self._prefetching = True
449 self._prefetch_done = False
450 self._prefetch_reads.extend(chunks)
451
452 t = threading.Thread(target=self._prefetch_thread, args=(chunks,))
453 t.setDaemon(True)
454 t.start()
455
457
458
459 for offset, length in chunks:
460 self.sftp._async_request(self, CMD_READ, self.handle, long(offset), int(length))
461
463 if t == CMD_STATUS:
464
465 try:
466 self.sftp._convert_status(msg)
467 except Exception, x:
468 self._saved_exception = x
469 return
470 if t != CMD_DATA:
471 raise SFTPError('Expected data')
472 data = msg.get_string()
473 offset, length = self._prefetch_reads.pop(0)
474 self._prefetch_data[offset] = data
475 if len(self._prefetch_reads) == 0:
476 self._prefetch_done = True
477
479 "if there's a saved exception, raise & clear it"
480 if self._saved_exception is not None:
481 x = self._saved_exception
482 self._saved_exception = None
483 raise x
484
487
488 - def __exit__(self, type, value, traceback):
490