| image: docker | |
| variables: | |
| DOCKER_HOST: tcp://docker:2375/ | |
| DOCKER_DRIVER: overlay2 | |
| DOCKER_TLS_CERTDIR: "" | |
| services: | |
| - docker:dind |
| import os | |
| import sys | |
| import traceback | |
| from airflow.models import DagBag | |
| from airflow.utils.session import provide_session | |
| from airflow.dag_processing.processor import DagFileProcessor | |
| DAG_DIRS = ['<path/to/dir1>', '<path/to/dir2>', ...] | |
| import os | |
| import logging | |
| import sys | |
| class YarnLogger: | |
| @staticmethod | |
| def setup_logger(): | |
| if not 'LOG_DIRS' in os.environ: | |
| sys.stderr.write('Missing LOG_DIRS environment variable, pyspark logging disabled') | |
| return |
| """ | |
| conda install -c conda-forge postgresql psycopg2 sqlalchemy | |
| initdb -D /tmp/dask_db | |
| postgresql -D /tmp/dask_db | |
| pg_ctl -D /tmp/dask_db -l logfile start | |
| createuser --encrypted --pwprompt dask | |
| createdb --owner=dask dask_db | |
| """ |
| { | |
| "workbench.colorTheme": "Default Light+", | |
| "workbench.iconTheme": "material-icon-theme", | |
| "workbench.sideBar.location": "right", | |
| "editor.fontFamily": "'Dank Mono', 'Fira Code', 'Operator Mono Lig Book', 'FantasqueSansMono Nerd Font Mono'", | |
| "editor.fontLigatures": true, | |
| "editor.fontSize": 15, | |
| "editor.detectIndentation": false, | |
| "editor.tabSize": 4, | |
| "[javascript]": { |
| # see also https://github.com/wrobstory/pgshift | |
| import gzip | |
| from io import StringIO, BytesIO | |
| from functools import wraps | |
| import boto | |
| from sqlalchemy import MetaData | |
| from pandas import DataFrame | |
| from pandas.io.sql import SQLTable, pandasSQL_builder |
I have a pet project I work on, every now and then. CNoEvil.
The concept is simple enough.
What if, for a moment, we forgot all the rules we know. That we ignore every good idea, and accept all the terrible ones. That nothing is off limits. Can we turn C into a new language? Can we do what Lisp and Forth let the over-eager programmer do, but in C?
| from datetime import datetime | |
| # Usage: | |
| # with S3MultipartUpload(client=boto3.client('s3'), destination_bucket, destination_key) as s3upload: | |
| # # Do something | |
| # s3upload.upload(content) | |
| # # Do something | |
| # s3upload.upload(content) | |
| # # Do something | |
| # s3upload.flush() |
This is a mix between two sources:
basically the first resource is great but didn't work for me: I had to remove the trailing "/*" in the resource string to make it work. I also noticed that setting the policy on the source bucket was sufficient. In the end these are the exact steps I followed to copy data between two buckets on two accounts
Basically the idea there is:
| <?xml version="1.0" encoding="UTF-8"?> | |
| <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> | |
| <plist version="1.0"> | |
| <dict> | |
| <key>ACPI</key> | |
| <dict> | |
| <key>Add</key> | |
| <array> | |
| <dict> | |
| <key>Comment</key> |
