Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
production
geocannabis--drone-analytics-2
azure-blob-lib
Commits
e5f73422
Commit
e5f73422
authored
Feb 02, 2021
by
Benjamin Hourte
Browse files
Adding the initial code
parent
eaf355aa
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
302 additions
and
0 deletions
+302
-0
.gitignore
.gitignore
+106
-0
azure_blob_client/__init__.py
azure_blob_client/__init__.py
+148
-0
azure_blob_client/__main__.py
azure_blob_client/__main__.py
+22
-0
azure_blob_client/utils.py
azure_blob_client/utils.py
+26
-0
No files found.
.gitignore
0 → 100644
View file @
e5f73422
# Created by https://www.gitignore.io/api/python, and modified...
.vscode/
node_modules/
### UNIX TEMP/BACKUP FILES ###
*~
### Mac OS X stuff ###
Desktop DF
Desktop DB
.Spotlight-V100
.DS_Store
.Trashes
.com.apple.timemachine.supported
.fseventsd
.syncinfo
.TemporaryItems
report/
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
/dist/
server/static/dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
MANIFEST
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# This App logs
*.log
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# IPython Notebook
.ipynb_checkpoints
# pyenv
.python-version
# Eclipse
.settings/
/pylint.out
/.idea/
/main-ui.iml
server/static/
server/templates/index.html
.pytest_cache/
venv/
.venv/
.project
docs/
server/development.py
package-lock.json
*.bak
.env
azure_blob_client/__init__.py
0 → 100644
View file @
e5f73422
# pylint: disable=line-too-long
"""Module implements The generic Blob Client for Azure
:raises NotImplementedError: Raised for when trying to upload a blob not of type BlockBlob
"""
from
os
import
remove
from
typing
import
ContextManager
,
Iterator
,
List
from
datetime
import
datetime
,
timedelta
import
logging
from
azure.storage.blob
import
(
BlobServiceClient
,
generate_account_sas
,
ResourceTypes
,
AccountSasPermissions
,
ContainerClient
,
BlobType
,
)
from
azure.core.exceptions
import
ClientAuthenticationError
LOGGER
=
logging
.
getLogger
()
class
AzureBlobClient
:
"""Generic Blob Client for Azure"""
def
__init__
(
self
,
storage_account
:
str
,
credentials
:
str
):
"""Builder of AzueBlobClient
:param storage_account: Parameter from Azure: the storage account
:type storage_account: str
:param credentials: Parameter from Azure: the key (credential)
:type credentials: str
"""
self
.
storage_account
=
storage_account
self
.
credentials
=
credentials
self
.
account_url
=
"https://{account_name}.blob.core.windows.net"
.
format
(
account_name
=
self
.
storage_account
)
self
.
sas_token
=
None
def
_generate_sas_signature
(
self
,
validity_in_secs
:
int
=
30
)
->
str
:
"""Generate an adapted SAS token
:param validity_in_secs: Validity for the token expressed in seconds, defaults to 30
:type validity_in_secs: int, optional
:return: SAS token
:rtype: str
"""
return
generate_account_sas
(
account_name
=
self
.
storage_account
,
account_key
=
self
.
credentials
,
resource_types
=
ResourceTypes
(
service
=
True
,
container
=
True
,
object
=
True
),
permission
=
AccountSasPermissions
(
read
=
True
,
list
=
True
,
create
=
True
,
add
=
True
,
remove
=
True
,
delete
=
True
),
expiry
=
datetime
.
utcnow
()
+
timedelta
(
seconds
=
validity_in_secs
),
)
def
blob_container_client
(
self
,
container_name
:
str
)
->
ContainerClient
:
"""Builder of container client
:param container_name: Name of the container
:type container_name: str
:return: Container Client with goog authentication
:rtype: ContainerClient
"""
return
ContainerClient
(
account_url
=
self
.
account_url
,
container_name
=
container_name
,
credential
=
self
.
_generate_sas_signature
()
)
def
blob_service_client
(
self
)
->
BlobServiceClient
:
"""Builder of service client
:return: Service Client with goog authentication
:rtype: BlobServiceClient
"""
return
BlobServiceClient
(
self
.
account_url
,
credential
=
self
.
_generate_sas_signature
())
def
list_containers
(
self
)
->
list
:
"""Function to retrieve the containers
:return: containers
:rtype: list
"""
containers
=
[]
for
container
in
self
.
blob_service_client
().
list_containers
():
containers
.
append
(
container
)
return
containers
def
list_blobs_in_container
(
self
,
container_name
:
str
)
->
list
:
"""Function to list the blobs present in a container
:param container: Name of the container to list
:type container: str
:return: list of the blobs
:rtype: list
"""
blobs
=
[]
for
blob
in
self
.
blob_container_client
(
container_name
=
container_name
).
list_blobs
():
blobs
.
append
(
blob
)
return
blobs
def
create_container
(
self
,
container_name
:
str
)
->
dict
:
"""Create a container within the account storage
:param container_name: Name of the container to create
:type container_name: str
:return: a description of the created container
:rtype: dict
"""
return
self
.
blob_service_client
().
create_container
(
name
=
container_name
)
def
delete_container
(
self
,
container_name
:
str
):
"""Function to remove a container within the account storage
:param container_name: Name of the container to remove
:type container_name: str
"""
return
self
.
blob_service_client
().
delete_container
(
container
=
container_name
)
def
upload_blob_in_container
(
self
,
container_name
:
str
,
blob_name
:
str
,
blob_filename
:
str
,
blob_type
:
BlobType
=
BlobType
.
BlockBlob
):
"""Upload a new blob in the container
:param container_name: Name of the container in which we need to send the new blob
:type container_name: str
:param blob_name: Name to give to the new blob in the container
:type blob_name: str
:param blob_content: The blob content
:type blob_content: bytes
"""
if
blob_type
!=
BlobType
.
BlockBlob
:
LOGGER
.
error
(
"Only the type 'BlockBlob' is supported for the moment"
)
raise
NotImplementedError
with
open
(
blob_filename
,
"rb"
)
as
data
:
self
.
blob_container_client
(
container_name
=
container_name
).
upload_blob
(
name
=
blob_name
,
data
=
data
,
blob_type
=
blob_type
)
def
delete_blob_in_container
(
self
,
container_name
:
str
,
blob_name
:
str
):
"""Function to delete a blob within a container
:param container_name: Name of the container
:type container_name: str
:param blob_name: Name of the blob to delete
:type blob_name: str
"""
self
.
blob_container_client
(
container_name
=
container_name
).
delete_blob
(
blob
=
blob_name
)
\ No newline at end of file
azure_blob_client/__main__.py
0 → 100644
View file @
e5f73422
# pylint: disable=line-too-long
"""Launcher for the azure_blob_client"""
import
os
import
logging
import
time
from
.
import
AzureBlobClient
from
.utils
import
setup_logging
STORAGE_ACCOUNT
=
"___storage_account___"
CREDENTIALS
=
"___credentials___"
if
__name__
==
"__main__"
:
## Initialisation of the logger
setup_logging
()
LOGGER
=
logging
.
getLogger
(
__name__
)
LOGGER
.
info
(
"Logger successfully initialised"
)
### Do your stuff here...
while
True
:
time
.
sleep
(
10
)
\ No newline at end of file
azure_blob_client/utils.py
0 → 100644
View file @
e5f73422
"""Different utils used for logging for example"""
import
os
import
logging
import
logging.config
import
yaml
def
setup_logging
(
default_path
=
"logging.yaml"
,
default_level
=
logging
.
INFO
,
env_key
=
"LOG_CFG"
):
"""Setup logging configuration
:param default_path: ogging config file, defaults to "logging.yaml"
:type default_path: str, optional
:param default_level: detail level for the logger, defaults to logging.INFO
:type default_level: str, optional
:param env_key: Configuration key, defaults to "LOG_CFG"
:type env_key: str, optional
"""
path
=
os
.
getenv
(
env_key
,
default_path
)
if
os
.
path
.
exists
(
path
):
with
open
(
path
,
"rt"
)
as
config_file
:
config
=
yaml
.
safe_load
(
config_file
.
read
())
logging
.
config
.
dictConfig
(
config
)
else
:
logging
.
basicConfig
(
level
=
default_level
)
logging
.
getLogger
(
"azure.core.pipeline.policies.http_logging_policy"
).
setLevel
(
logging
.
WARNING
)
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment