LSSTApplications  19.0.0-14-gb0260a2+72efe9b372,20.0.0+7927753e06,20.0.0+8829bf0056,20.0.0+995114c5d2,20.0.0+b6f4b2abd1,20.0.0+bddc4f4cbe,20.0.0-1-g253301a+8829bf0056,20.0.0-1-g2b7511a+0d71a2d77f,20.0.0-1-g5b95a8c+7461dd0434,20.0.0-12-g321c96ea+23efe4bbff,20.0.0-16-gfab17e72e+fdf35455f6,20.0.0-2-g0070d88+ba3ffc8f0b,20.0.0-2-g4dae9ad+ee58a624b3,20.0.0-2-g61b8584+5d3db074ba,20.0.0-2-gb780d76+d529cf1a41,20.0.0-2-ged6426c+226a441f5f,20.0.0-2-gf072044+8829bf0056,20.0.0-2-gf1f7952+ee58a624b3,20.0.0-20-geae50cf+e37fec0aee,20.0.0-25-g3dcad98+544a109665,20.0.0-25-g5eafb0f+ee58a624b3,20.0.0-27-g64178ef+f1f297b00a,20.0.0-3-g4cc78c6+e0676b0dc8,20.0.0-3-g8f21e14+4fd2c12c9a,20.0.0-3-gbd60e8c+187b78b4b8,20.0.0-3-gbecbe05+48431fa087,20.0.0-38-ge4adf513+a12e1f8e37,20.0.0-4-g97dc21a+544a109665,20.0.0-4-gb4befbc+087873070b,20.0.0-4-gf910f65+5d3db074ba,20.0.0-5-gdfe0fee+199202a608,20.0.0-5-gfbfe500+d529cf1a41,20.0.0-6-g64f541c+d529cf1a41,20.0.0-6-g9a5b7a1+a1cd37312e,20.0.0-68-ga3f3dda+5fca18c6a4,20.0.0-9-g4aef684+e18322736b,w.2020.45
LSSTDataManagementBasePackage
safeFileIo.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 """
24 Utilities for safe file IO
25 """
26 from contextlib import contextmanager
27 import errno
28 import fcntl
29 import filecmp
30 import os
31 import tempfile
32 from lsst.log import Log
33 
34 
35 class DoNotWrite(RuntimeError):
36  pass
37 
38 
39 def safeMakeDir(directory):
40  """Make a directory in a manner avoiding race conditions"""
41  if directory != "" and not os.path.exists(directory):
42  try:
43  os.makedirs(directory)
44  except OSError as e:
45  # Don't fail if directory exists due to race
46  if e.errno != errno.EEXIST:
47  raise e
48 
49 
50 def setFileMode(filename):
51  """Set a file mode according to the user's umask"""
52  # Get the current umask, which we can only do by setting it and then reverting to the original.
53  umask = os.umask(0o077)
54  os.umask(umask)
55  # chmod the new file to match what it would have been if it hadn't started life as a temporary
56  # file (which have more restricted permissions).
57  os.chmod(filename, (~umask & 0o666))
58 
59 
61  pass
62 
63 
64 @contextmanager
66  """Context manager to get a file that can be written only once and all other writes will succeed only if
67  they match the initial write.
68 
69  The context manager provides a temporary file object. After the user is done, the temporary file becomes
70  the permanent file if the file at name does not already exist. If the file at name does exist the
71  temporary file is compared to the file at name. If they are the same then this is good and the temp file
72  is silently thrown away. If they are not the same then a runtime error is raised.
73  """
74  outDir, outName = os.path.split(name)
75  safeMakeDir(outDir)
76  temp = tempfile.NamedTemporaryFile(mode="w", dir=outDir, prefix=outName, delete=False)
77  try:
78  yield temp
79  finally:
80  try:
81  temp.close()
82  # If the symlink cannot be created then it will raise. If it can't be created because a file at
83  # 'name' already exists then we'll do a compare-same check.
84  os.symlink(temp.name, name)
85  # If the symlink was created then this is the process that created the first instance of the
86  # file, and we know its contents match. Move the temp file over the symlink.
87  os.rename(temp.name, name)
88  # At this point, we know the file has just been created. Set permissions according to the
89  # current umask.
90  setFileMode(name)
91  except OSError as e:
92  if e.errno != errno.EEXIST:
93  raise e
94  filesMatch = filecmp.cmp(temp.name, name, shallow=False)
95  os.remove(temp.name)
96  if filesMatch:
97  # if the files match then the compare-same check succeeded and we can silently return.
98  return
99  else:
100  # if the files do not match then the calling code was trying to write a non-matching file over
101  # the previous file, maybe it's a race condition? In any event, raise a runtime error.
102  raise FileForWriteOnceCompareSameFailure("Written file does not match existing file.")
103 
104 
105 @contextmanager
106 def SafeFile(name):
107  """Context manager to create a file in a manner avoiding race conditions
108 
109  The context manager provides a temporary file object. After the user is done,
110  we move that file into the desired place and close the fd to avoid resource
111  leakage.
112  """
113  outDir, outName = os.path.split(name)
114  safeMakeDir(outDir)
115  doWrite = True
116  with tempfile.NamedTemporaryFile(mode="w", dir=outDir, prefix=outName, delete=False) as temp:
117  try:
118  yield temp
119  except DoNotWrite:
120  doWrite = False
121  finally:
122  if doWrite:
123  os.rename(temp.name, name)
124  setFileMode(name)
125 
126 
127 @contextmanager
128 def SafeFilename(name):
129  """Context manager for creating a file in a manner avoiding race conditions
130 
131  The context manager provides a temporary filename with no open file descriptors
132  (as this can cause trouble on some systems). After the user is done, we move the
133  file into the desired place.
134  """
135  outDir, outName = os.path.split(name)
136  safeMakeDir(outDir)
137  temp = tempfile.NamedTemporaryFile(mode="w", dir=outDir, prefix=outName, delete=False)
138  tempName = temp.name
139  temp.close() # We don't use the fd, just want a filename
140  try:
141  yield tempName
142  finally:
143  os.rename(tempName, name)
144  setFileMode(name)
145 
146 
147 @contextmanager
149  """Context manager for reading a file that may be locked with an exclusive lock via
150  SafeLockedFileForWrite. This will first acquire a shared lock before returning the file. When the file is
151  closed the shared lock will be unlocked.
152 
153  Parameters
154  ----------
155  name : string
156  The file name to be opened, may include path.
157 
158  Yields
159  ------
160  file object
161  The file to be read from.
162  """
163  log = Log.getLogger("daf.persistence.butler")
164  try:
165  with open(name, 'r') as f:
166  log.debug("Acquiring shared lock on {}".format(name))
167  fcntl.flock(f, fcntl.LOCK_SH)
168  log.debug("Acquired shared lock on {}".format(name))
169  yield f
170  finally:
171  log.debug("Releasing shared lock on {}".format(name))
172 
173 
175  """File-like object that is used to create a file if needed, lock it with an exclusive lock, and contain
176  file descriptors to readable and writable versions of the file.
177 
178  This will only open a file descriptor in 'write' mode if a write operation is performed. If no write
179  operation is performed, the existing file (if there is one) will not be overwritten.
180 
181  Contains __enter__ and __exit__ functions so this can be used by a context manager.
182  """
183  def __init__(self, name):
184  self.log = Log.getLogger("daf.persistence.butler")
185  self.name = name
186  self._readable = None
187  self._writeable = None
188  safeMakeDir(os.path.split(name)[0])
189 
190  def __enter__(self):
191  self.open()
192  return self
193 
194  def __exit__(self, type, value, traceback):
195  self.close()
196 
197  def open(self):
198  self._fileHandle = open(self.name, 'a')
199  self.log.debug("Acquiring exclusive lock on {}".format(self.name))
200  fcntl.flock(self._fileHandle, fcntl.LOCK_EX)
201  self.log.debug("Acquired exclusive lock on {}".format(self.name))
202 
203  def close(self):
204  self.log.debug("Releasing exclusive lock on {}".format(self.name))
205  if self._writeable is not None:
206  self._writeable.close()
207  if self._readable is not None:
208  self._readable.close()
209  self._fileHandle.close()
210 
211  @property
212  def readable(self):
213  if self._readable is None:
214  self._readable = open(self.name, 'r')
215  return self._readable
216 
217  @property
218  def writeable(self):
219  if self._writeable is None:
220  self._writeable = open(self.name, 'w')
221  return self._writeable
222 
223  def read(self, size=None):
224  if size is not None:
225  return self.readable.read(size)
226  return self.readable.read()
227 
228  def write(self, str):
229  self.writeable.write(str)
lsst::daf::persistence.safeFileIo.setFileMode
def setFileMode(filename)
Definition: safeFileIo.py:50
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.__init__
def __init__(self, name)
Definition: safeFileIo.py:183
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.open
def open(self)
Definition: safeFileIo.py:197
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.__enter__
def __enter__(self)
Definition: safeFileIo.py:190
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite._readable
_readable
Definition: safeFileIo.py:186
lsst.gdb.afw.printers.debug
bool debug
Definition: printers.py:9
lsst::daf::persistence.safeFileIo.DoNotWrite
Definition: safeFileIo.py:35
lsst::daf::persistence.safeFileIo.FileForWriteOnceCompareSameFailure
Definition: safeFileIo.py:60
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite
Definition: safeFileIo.py:174
lsst::daf::persistence.safeFileIo.safeMakeDir
def safeMakeDir(directory)
Definition: safeFileIo.py:39
lsst::daf::persistence.safeFileIo.SafeFilename
def SafeFilename(name)
Definition: safeFileIo.py:128
lsst.pex.config.history.format
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
lsst::daf::persistence.safeFileIo.SafeLockedFileForRead
def SafeLockedFileForRead(name)
Definition: safeFileIo.py:148
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.__exit__
def __exit__(self, type, value, traceback)
Definition: safeFileIo.py:194
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.readable
def readable(self)
Definition: safeFileIo.py:212
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite._fileHandle
_fileHandle
Definition: safeFileIo.py:198
lsst::daf::persistence.safeFileIo.FileForWriteOnceCompareSame
def FileForWriteOnceCompareSame(name)
Definition: safeFileIo.py:65
lsst::log
Definition: Log.h:706
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.write
def write(self, str)
Definition: safeFileIo.py:228
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite._writeable
_writeable
Definition: safeFileIo.py:187
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.writeable
def writeable(self)
Definition: safeFileIo.py:218
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.name
name
Definition: safeFileIo.py:185
lsst::daf::persistence.safeFileIo.SafeFile
def SafeFile(name)
Definition: safeFileIo.py:106
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.close
def close(self)
Definition: safeFileIo.py:203
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.log
log
Definition: safeFileIo.py:184
lsst::daf::persistence.safeFileIo.SafeLockedFileForWrite.read
def read(self, size=None)
Definition: safeFileIo.py:223