Python azure.common.AzureMissingResourceHttpError() Examples
The following are 30
code examples of azure.common.AzureMissingResourceHttpError().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
azure.common
, or try the search function
.
.
Example #1
| Source File: archive.py From bob with GNU General Public License v3.0 | 6 votes |
|
def _openDownloadFile(self, buildId, suffix):
from azure.common import AzureException, AzureMissingResourceHttpError
(tmpFd, tmpName) = mkstemp()
try:
os.close(tmpFd)
self.__service.get_blob_to_path(self.__container,
self.__makeBlobName(buildId, suffix), tmpName)
ret = tmpName
tmpName = None
return AzureDownloader(ret)
except AzureMissingResourceHttpError:
raise ArtifactNotFoundError()
except AzureException as e:
raise ArtifactDownloadError(str(e))
finally:
if tmpName is not None: os.unlink(tmpName)
Example #2
| Source File: monitor.py From pan-fca with Apache License 2.0 | 6 votes |
|
def get_vm_in_cosmos_db(self, spoke, vm_hostname):
try:
db_vm_info = self.table_service.get_entity(self.vmss_table_name,
spoke, vm_hostname)
except AzureMissingResourceHttpError:
self.logger.info("New VM %s found in spoke %s" % (vm_hostname, spoke))
return None
except Exception as e:
self.logger.error("Querying for %s failed" % vm_hostname)
return None
else:
# IF possible update status TODO
self.logger.debug("VM %s is available in VMSS, Pan and DB" % (vm_hostname))
return db_vm_info
# 'name' : global_device['@name'],
# 'hostname' : global_device['hostname'],
# 'serial' : global_device['serial'],
# 'ip-address' : global_device['ip-address'],
# 'connected' : global_device['connected'],
# 'deactivated': global_device['deactivated']
Example #3
| Source File: snapshot.py From scyllabackup with MIT License | 6 votes |
|
def file_delete_worker(self):
"""Worker for deleting files. This worker is mapped to gevent threads
for concurrency. Number of threads is configured by `self.max_workers`
:returns: Nothing
:rtype: None
"""
while True:
try:
# file_tuple = tuple(keyspace,tablename,file)
file_tuple = self._delete_queue.get()
storage_key = '/'.join((self._prefix, ) + file_tuple)
self._storage.delete_key(storage_key)
except AzureMissingResourceHttpError as e:
logger.error("Deletion of blob {0} failed. It's already deleted or missing.".format(storage_key))
except Exception as e:
logger.exception("Unexpected exception encountered")
sys.exit(4)
finally:
self._delete_queue.task_done()
Example #4
| Source File: azure_blob_writer.py From exporters with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def _check_write_consistency(self):
from azure.common import AzureMissingResourceHttpError
for blob_info in self.get_metadata('blobs_written'):
try:
blob = self.azure_service.get_blob_properties(
self.read_option('container'), blob_info['blob_name'])
blob_size = blob.properties.content_length
blob_md5 = blob.properties.content_settings.content_md5
if str(blob_size) != str(blob_info['size']):
raise InconsistentWriteState(
'File {} has unexpected size. (expected {} - got {})'.format(
blob_info['blob_name'], blob_info['size'], blob_size
)
)
if str(blob_md5) != str(blob_info['hash']):
raise InconsistentWriteState(
'File {} has unexpected hash. (expected {} - got {})'.format(
blob_info['blob_name'], blob_info['hash'], blob_md5
)
)
except AzureMissingResourceHttpError:
raise InconsistentWriteState('Missing blob {}'.format(blob_info['blob_name']))
self.logger.info('Consistency check passed')
Example #5
| Source File: test_writers_azure.py From exporters with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def test_write_file_consistency_present(self, create_mock, create_share_mock,
put_file_from_path_mock, get_file_properties_mock):
from azure.common import AzureMissingResourceHttpError
# given
items_to_write = self.get_batch()
options = self.get_writer_config()
options['options']['check_consistency'] = True
get_file_properties_mock.side_effect = AzureMissingResourceHttpError('', 404)
# when:
writer = AzureFileWriter(options, meta())
try:
writer.write_batch(items_to_write)
writer.flush()
finally:
writer.close()
with self.assertRaisesRegexp(InconsistentWriteState, 'Missing file'):
writer.finish_writing()
# then:
self.assertEqual(writer.get_metadata('items_count'), 2)
Example #6
| Source File: test_writers_azure.py From exporters with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def test_write_blob_consistency_present(self, create_mock, create_blob_from_path_mock,
get_blob_properties_mock):
from azure.common import AzureMissingResourceHttpError
# given
items_to_write = self.get_batch()
options = self.get_writer_config()
options['options']['check_consistency'] = True
get_blob_properties_mock.side_effect = AzureMissingResourceHttpError('', 404)
# when:
writer = AzureBlobWriter(options, meta())
try:
writer.write_batch(items_to_write)
writer.flush()
finally:
writer.close()
with self.assertRaisesRegexp(InconsistentWriteState, 'Missing blob'):
writer.finish_writing()
Example #7
| Source File: azure_utils.py From BackpropThroughTheVoidRL with MIT License | 5 votes |
|
def exists(self, blob_name):
"""Returns true if `blob_name` exists in container."""
try:
self._service.get_blob_properties(
blob_name=blob_name,
container_name=self._container_name
)
return True
except AzureMissingResourceHttpError:
return False
Example #8
| Source File: azure_blob_storage_helper.py From healthcareai-py with MIT License | 5 votes |
|
def create_container(self, container_name):
"""
Create a new container on Azure if it does not exist.
:param container_name: the name of the container
"""
try:
return self._connection.create_container(container_name)
except AzureMissingResourceHttpError:
raise AzureBlobStorageHelperError('The specified container does not exist.')
Example #9
| Source File: azure_utils.py From BackpropThroughTheVoidRL with MIT License | 5 votes |
|
def put(self, source_path, blob_name, callback=None):
"""Upload a file or directory from `source_path` to azure blob `blob_name`.
Upload progress can be traced by an optional callback.
"""
upload_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
upload_done.set()
# Attempt to make backup if an existing version is already available
try:
x_ms_copy_source = "https://{}.blob.core.windows.net/{}/{}".format(
self._account_name,
self._container_name,
blob_name
)
self._service.copy_blob(
container_name=self._container_name,
blob_name=blob_name + ".backup",
x_ms_copy_source=x_ms_copy_source
)
except AzureMissingResourceHttpError:
pass
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
make_archive(source_path, arcpath)
self._service.put_block_blob_from_path(
container_name=self._container_name,
blob_name=blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
upload_done.wait()
Example #10
| Source File: azure_utils.py From emdqn with MIT License | 5 votes |
|
def exists(self, blob_name):
"""Returns true if `blob_name` exists in container."""
try:
self._service.get_blob_properties(
blob_name=blob_name,
container_name=self._container_name
)
return True
except AzureMissingResourceHttpError:
return False
Example #11
| Source File: azure_utils.py From emdqn with MIT License | 5 votes |
|
def get(self, dest_path, blob_name, callback=None):
"""Download a file or directory to `dest_path` to azure blob `blob_name`.
Warning! If directory is downloaded the `dest_path` is the parent directory.
Upload progress can be traced by an optional callback.
"""
download_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
download_done.set()
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
for backup_blob_name in [blob_name, blob_name + '.backup']:
try:
blob_size = self._service.get_blob_properties(
blob_name=backup_blob_name,
container_name=self._container_name
)['content-length']
if int(blob_size) > 0:
self._service.get_blob_to_path(
container_name=self._container_name,
blob_name=backup_blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
unpack_archive(arcpath, dest_path)
download_done.wait()
return True
except AzureMissingResourceHttpError:
pass
return False
Example #12
| Source File: azure_utils.py From emdqn with MIT License | 5 votes |
|
def put(self, source_path, blob_name, callback=None):
"""Upload a file or directory from `source_path` to azure blob `blob_name`.
Upload progress can be traced by an optional callback.
"""
upload_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
upload_done.set()
# Attempt to make backup if an existing version is already available
try:
x_ms_copy_source = "https://{}.blob.core.windows.net/{}/{}".format(
self._account_name,
self._container_name,
blob_name
)
self._service.copy_blob(
container_name=self._container_name,
blob_name=blob_name + ".backup",
x_ms_copy_source=x_ms_copy_source
)
except AzureMissingResourceHttpError:
pass
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
make_archive(source_path, arcpath)
self._service.put_block_blob_from_path(
container_name=self._container_name,
blob_name=blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
upload_done.wait()
Example #13
| Source File: azure_utils.py From deeprl-baselines with MIT License | 5 votes |
|
def exists(self, blob_name):
"""Returns true if `blob_name` exists in container."""
try:
self._service.get_blob_properties(
blob_name=blob_name,
container_name=self._container_name
)
return True
except AzureMissingResourceHttpError:
return False
Example #14
| Source File: azure_utils.py From deeprl-baselines with MIT License | 5 votes |
|
def put(self, source_path, blob_name, callback=None):
"""Upload a file or directory from `source_path` to azure blob `blob_name`.
Upload progress can be traced by an optional callback.
"""
upload_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
upload_done.set()
# Attempt to make backup if an existing version is already available
try:
x_ms_copy_source = "https://{}.blob.core.windows.net/{}/{}".format(
self._account_name,
self._container_name,
blob_name
)
self._service.copy_blob(
container_name=self._container_name,
blob_name=blob_name + ".backup",
x_ms_copy_source=x_ms_copy_source
)
except AzureMissingResourceHttpError:
pass
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
make_archive(source_path, arcpath)
self._service.put_block_blob_from_path(
container_name=self._container_name,
blob_name=blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
upload_done.wait()
Example #15
| Source File: azure_storage.py From django-storages with BSD 3-Clause "New" or "Revised" License | 5 votes |
|
def delete(self, name):
try:
self.service.delete_blob(
container_name=self.azure_container,
blob_name=self._get_valid_path(name),
timeout=self.timeout)
except AzureMissingResourceHttpError:
pass
Example #16
| Source File: azure_file_writer.py From exporters with BSD 3-Clause "New" or "Revised" License | 5 votes |
|
def _check_write_consistency(self):
from azure.common import AzureMissingResourceHttpError
for file_info in self.get_metadata('files_written'):
try:
afile = self.azure_service.get_file_properties(
self.share, file_info['filebase_path'], file_info['file_name'])
file_size = afile.properties.content_length
if str(file_size) != str(file_info['size']):
raise InconsistentWriteState(
'File {} has unexpected size. (expected {} - got {})'.format(
file_info['file_name'], file_info['size'], file_size)
)
except AzureMissingResourceHttpError:
raise InconsistentWriteState('Missing file {}'.format(file_info['file_name']))
self.logger.info('Consistency check passed')
Example #17
| Source File: azure_utils.py From learning2run with MIT License | 5 votes |
|
def put(self, source_path, blob_name, callback=None):
"""Upload a file or directory from `source_path` to azure blob `blob_name`.
Upload progress can be traced by an optional callback.
"""
upload_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
upload_done.set()
# Attempt to make backup if an existing version is already available
try:
x_ms_copy_source = "https://{}.blob.core.windows.net/{}/{}".format(
self._account_name,
self._container_name,
blob_name
)
self._service.copy_blob(
container_name=self._container_name,
blob_name=blob_name + ".backup",
x_ms_copy_source=x_ms_copy_source
)
except AzureMissingResourceHttpError:
pass
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
make_archive(source_path, arcpath)
self._service.put_block_blob_from_path(
container_name=self._container_name,
blob_name=blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
upload_done.wait()
Example #18
| Source File: azure_utils.py From rl-attack with MIT License | 5 votes |
|
def put(self, source_path, blob_name, callback=None):
"""Upload a file or directory from `source_path` to azure blob `blob_name`.
Upload progress can be traced by an optional callback.
"""
upload_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
upload_done.set()
# Attempt to make backup if an existing version is already available
try:
x_ms_copy_source = "https://{}.blob.core.windows.net/{}/{}".format(
self._account_name,
self._container_name,
blob_name
)
self._service.copy_blob(
container_name=self._container_name,
blob_name=blob_name + ".backup",
x_ms_copy_source=x_ms_copy_source
)
except AzureMissingResourceHttpError:
pass
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
make_archive(source_path, arcpath)
self._service.put_block_blob_from_path(
container_name=self._container_name,
blob_name=blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
upload_done.wait()
Example #19
| Source File: azure_utils.py From action-branching-agents with MIT License | 5 votes |
|
def exists(self, blob_name):
"""Returns true if `blob_name` exists in container."""
try:
self._service.get_blob_properties(
blob_name=blob_name,
container_name=self._container_name
)
return True
except AzureMissingResourceHttpError:
return False
Example #20
| Source File: azure_utils.py From action-branching-agents with MIT License | 5 votes |
|
def get(self, dest_path, blob_name, callback=None):
"""Download a file or directory to `dest_path` to azure blob `blob_name`.
Warning! If directory is downloaded the `dest_path` is the parent directory.
Upload progress can be traced by an optional callback.
"""
download_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
download_done.set()
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
for backup_blob_name in [blob_name, blob_name + '.backup']:
try:
blob_size = self._service.get_blob_properties(
blob_name=backup_blob_name,
container_name=self._container_name
)['content-length']
if int(blob_size) > 0:
self._service.get_blob_to_path(
container_name=self._container_name,
blob_name=backup_blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
unpack_archive(arcpath, dest_path)
download_done.wait()
return True
except AzureMissingResourceHttpError:
pass
return False
Example #21
| Source File: azure_utils.py From action-branching-agents with MIT License | 5 votes |
|
def put(self, source_path, blob_name, callback=None):
"""Upload a file or directory from `source_path` to azure blob `blob_name`.
Upload progress can be traced by an optional callback.
"""
upload_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
upload_done.set()
# Attempt to make backup if an existing version is already available
try:
x_ms_copy_source = "https://{}.blob.core.windows.net/{}/{}".format(
self._account_name,
self._container_name,
blob_name
)
self._service.copy_blob(
container_name=self._container_name,
blob_name=blob_name + ".backup",
x_ms_copy_source=x_ms_copy_source
)
except AzureMissingResourceHttpError:
pass
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
make_archive(source_path, arcpath)
self._service.put_block_blob_from_path(
container_name=self._container_name,
blob_name=blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
upload_done.wait()
Example #22
| Source File: azure_utils.py From rl-attack-detection with MIT License | 5 votes |
|
def exists(self, blob_name):
"""Returns true if `blob_name` exists in container."""
try:
self._service.get_blob_properties(
blob_name=blob_name,
container_name=self._container_name
)
return True
except AzureMissingResourceHttpError:
return False
Example #23
| Source File: azure_utils.py From rl-attack-detection with MIT License | 5 votes |
|
def get(self, dest_path, blob_name, callback=None):
"""Download a file or directory to `dest_path` to azure blob `blob_name`.
Warning! If directory is downloaded the `dest_path` is the parent directory.
Upload progress can be traced by an optional callback.
"""
download_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
download_done.set()
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
for backup_blob_name in [blob_name, blob_name + '.backup']:
try:
blob_size = self._service.get_blob_properties(
blob_name=backup_blob_name,
container_name=self._container_name
)['content-length']
if int(blob_size) > 0:
self._service.get_blob_to_path(
container_name=self._container_name,
blob_name=backup_blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
unpack_archive(arcpath, dest_path)
download_done.wait()
return True
except AzureMissingResourceHttpError:
pass
return False
Example #24
| Source File: azure_utils.py From rl-attack with MIT License | 5 votes |
|
def exists(self, blob_name):
"""Returns true if `blob_name` exists in container."""
try:
self._service.get_blob_properties(
blob_name=blob_name,
container_name=self._container_name
)
return True
except AzureMissingResourceHttpError:
return False
Example #25
| Source File: azure_utils.py From rl-attack-detection with MIT License | 5 votes |
|
def put(self, source_path, blob_name, callback=None):
"""Upload a file or directory from `source_path` to azure blob `blob_name`.
Upload progress can be traced by an optional callback.
"""
upload_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
upload_done.set()
# Attempt to make backup if an existing version is already available
try:
x_ms_copy_source = "https://{}.blob.core.windows.net/{}/{}".format(
self._account_name,
self._container_name,
blob_name
)
self._service.copy_blob(
container_name=self._container_name,
blob_name=blob_name + ".backup",
x_ms_copy_source=x_ms_copy_source
)
except AzureMissingResourceHttpError:
pass
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
make_archive(source_path, arcpath)
self._service.put_block_blob_from_path(
container_name=self._container_name,
blob_name=blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
upload_done.wait()
Example #26
| Source File: azure_utils.py From gail-tf with MIT License | 5 votes |
|
def exists(self, blob_name):
"""Returns true if `blob_name` exists in container."""
try:
self._service.get_blob_properties(
blob_name=blob_name,
container_name=self._container_name
)
return True
except AzureMissingResourceHttpError:
return False
Example #27
| Source File: azure_utils.py From gail-tf with MIT License | 5 votes |
|
def put(self, source_path, blob_name, callback=None):
"""Upload a file or directory from `source_path` to azure blob `blob_name`.
Upload progress can be traced by an optional callback.
"""
upload_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
upload_done.set()
# Attempt to make backup if an existing version is already available
try:
x_ms_copy_source = "https://{}.blob.core.windows.net/{}/{}".format(
self._account_name,
self._container_name,
blob_name
)
self._service.copy_blob(
container_name=self._container_name,
blob_name=blob_name + ".backup",
x_ms_copy_source=x_ms_copy_source
)
except AzureMissingResourceHttpError:
pass
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
make_archive(source_path, arcpath)
self._service.put_block_blob_from_path(
container_name=self._container_name,
blob_name=blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback,
max_retries=10)
upload_done.wait()
Example #28
| Source File: azure_utils.py From learning2run with MIT License | 5 votes |
|
def exists(self, blob_name):
"""Returns true if `blob_name` exists in container."""
try:
self._service.get_blob_properties(
blob_name=blob_name,
container_name=self._container_name
)
return True
except AzureMissingResourceHttpError:
return False
Example #29
| Source File: azure_utils.py From deeprl-baselines with MIT License | 4 votes |
|
def get(self, dest_path, blob_name, callback=None):
"""Download a file or directory to `dest_path` to azure blob `blob_name`.
Warning! If directory is downloaded the `dest_path` is the parent directory.
Upload progress can be traced by an optional callback.
"""
download_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
download_done.set()
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
for backup_blob_name in [blob_name, blob_name + '.backup']:
try:
properties = self._service.get_blob_properties(
blob_name=backup_blob_name,
container_name=self._container_name
)
if hasattr(properties, 'properties'):
# Annoyingly, Azure has changed the API and this now returns a blob
# instead of it's properties with up-to-date azure package.
blob_size = properties.properties.content_length
else:
blob_size = properties['content-length']
if int(blob_size) > 0:
self._service.get_blob_to_path(
container_name=self._container_name,
blob_name=backup_blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback)
unpack_archive(arcpath, dest_path)
download_done.wait()
return True
except AzureMissingResourceHttpError:
pass
return False
Example #30
| Source File: azure_utils.py From BackpropThroughTheVoidRL with MIT License | 4 votes |
|
def get(self, dest_path, blob_name, callback=None):
"""Download a file or directory to `dest_path` to azure blob `blob_name`.
Warning! If directory is downloaded the `dest_path` is the parent directory.
Upload progress can be traced by an optional callback.
"""
download_done = Event()
def progress_callback(current, total):
if callback:
callback(current, total)
if current >= total:
download_done.set()
with tempfile.TemporaryDirectory() as td:
arcpath = os.path.join(td, "archive.zip")
for backup_blob_name in [blob_name, blob_name + '.backup']:
try:
properties = self._service.get_blob_properties(
blob_name=backup_blob_name,
container_name=self._container_name
)
if hasattr(properties, 'properties'):
# Annoyingly, Azure has changed the API and this now returns a blob
# instead of it's properties with up-to-date azure package.
blob_size = properties.properties.content_length
else:
blob_size = properties['content-length']
if int(blob_size) > 0:
self._service.get_blob_to_path(
container_name=self._container_name,
blob_name=backup_blob_name,
file_path=arcpath,
max_connections=4,
progress_callback=progress_callback)
unpack_archive(arcpath, dest_path)
download_done.wait()
return True
except AzureMissingResourceHttpError:
pass
return False
