linnealovespie/data-engineering-zoomcamp
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: linnealovespie:stream-homework
Branches Tags
linnealovespie:main linnealovespie:stream-homework linnealovespie:homework-solution linnealovespie:de-zoomcamp-videos linnealovespie:de-zoomcamp linnealovespie:LO_Module1
..
compare: linnealovespie:LO_Module1
Branches Tags
linnealovespie:main linnealovespie:stream-homework linnealovespie:homework-solution linnealovespie:de-zoomcamp-videos linnealovespie:de-zoomcamp linnealovespie:LO_Module1

3 Commits
stream-hom ... LO_Module1

Author SHA1 Message Date
Luis Oliveira
c7068244ba Changing folder 2023-12-19 18:06:54 +00:00
Luis Oliveira
36f0cd6bd1 Merge branch 'main' of https://github.com/DataTalksClub/data-engineering-zoomcamp into LO_Module1 2023-12-19 18:04:35 +00:00
Luis Oliveira
af0223054c Adding 2023-12-19 18:04:00 +00:00
197 changed files with 1080 additions and 22114 deletions
Expand all files Collapse all files

938
01-docker-terraform/2_docker_sql/data-loading-parquet.ipynb
View File

@ -1,938 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "52bad16a",
"metadata": {},
"source": [
"# Data loading \n",
"\n",
"Here we will be using the ```.paraquet``` file we downloaded and do the following:\n",
" - Check metadata and table datatypes of the paraquet file/table\n",
" - Convert the paraquet file to pandas dataframe and check the datatypes. Additionally check the data dictionary to make sure you have the right datatypes in pandas, as pandas will automatically create the table in our database.\n",
" - Generate the DDL CREATE statement from pandas for a sanity check.\n",
" - Create a connection to our database using SQLAlchemy\n",
" - Convert our huge paraquet file into a iterable that has batches of 100,000 rows and load it into our database."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "afef2456",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-03T23:55:14.141738Z",
"start_time": "2023-12-03T23:55:14.124217Z"
}
},
"outputs": [],
"source": [
"import pandas as pd \n",
"import pyarrow.parquet as pq\n",
"from time import time"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c750d1d4",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-03T02:54:01.925350Z",
"start_time": "2023-12-03T02:54:01.661119Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"<pyarrow._parquet.FileMetaData object at 0x7fed89ffa540>\n",
" created_by: parquet-cpp-arrow version 13.0.0\n",
" num_columns: 19\n",
" num_rows: 2846722\n",
" num_row_groups: 3\n",
" format_version: 2.6\n",
" serialized_size: 6357"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Read metadata \n",
"pq.read_metadata('yellow_tripdata_2023-09.parquet')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a970fcf0",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-03T23:28:08.411945Z",
"start_time": "2023-12-03T23:28:08.177693Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"VendorID: int32\n",
"tpep_pickup_datetime: timestamp[us]\n",
"tpep_dropoff_datetime: timestamp[us]\n",
"passenger_count: int64\n",
"trip_distance: double\n",
"RatecodeID: int64\n",
"store_and_fwd_flag: large_string\n",
"PULocationID: int32\n",
"DOLocationID: int32\n",
"payment_type: int64\n",
"fare_amount: double\n",
"extra: double\n",
"mta_tax: double\n",
"tip_amount: double\n",
"tolls_amount: double\n",
"improvement_surcharge: double\n",
"total_amount: double\n",
"congestion_surcharge: double\n",
"Airport_fee: double"
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Read file, read the table from file and check schema\n",
"file = pq.ParquetFile('yellow_tripdata_2023-09.parquet')\n",
"table = file.read()\n",
"table.schema"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "43f6ea7e",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-03T23:28:22.870376Z",
"start_time": "2023-12-03T23:28:22.563414Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
"RangeIndex: 2846722 entries, 0 to 2846721\n",
"Data columns (total 19 columns):\n",
" # Column Dtype \n",
"--- ------ ----- \n",
" 0 VendorID int32 \n",
" 1 tpep_pickup_datetime datetime64[ns]\n",
" 2 tpep_dropoff_datetime datetime64[ns]\n",
" 3 passenger_count float64 \n",
" 4 trip_distance float64 \n",
" 5 RatecodeID float64 \n",
" 6 store_and_fwd_flag object \n",
" 7 PULocationID int32 \n",
" 8 DOLocationID int32 \n",
" 9 payment_type int64 \n",
" 10 fare_amount float64 \n",
" 11 extra float64 \n",
" 12 mta_tax float64 \n",
" 13 tip_amount float64 \n",
" 14 tolls_amount float64 \n",
" 15 improvement_surcharge float64 \n",
" 16 total_amount float64 \n",
" 17 congestion_surcharge float64 \n",
" 18 Airport_fee float64 \n",
"dtypes: datetime64[ns](2), float64(12), int32(3), int64(1), object(1)\n",
"memory usage: 380.1+ MB\n"
]
}
],
"source": [
"# Convert to pandas and check data \n",
"df = table.to_pandas()\n",
"df.info()"
]
},
{
"cell_type": "markdown",
"id": "ccf039a0",
"metadata": {},
"source": [
"We need to first create the connection to our postgres database. We can feed the connection information to generate the CREATE SQL query for the specific server. SQLAlchemy supports a variety of servers."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "44e701ae",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-03T22:50:25.811951Z",
"start_time": "2023-12-03T22:50:25.393987Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"<sqlalchemy.engine.base.Connection at 0x7fed98ea3190>"
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Create an open SQL database connection object or a SQLAlchemy connectable\n",
"from sqlalchemy import create_engine\n",
"\n",
"engine = create_engine('postgresql://root:root@localhost:5432/ny_taxi')\n",
"engine.connect()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c96a1075",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-03T22:50:43.628727Z",
"start_time": "2023-12-03T22:50:43.442337Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"CREATE TABLE yellow_taxi_data (\n",
"\t\"VendorID\" INTEGER, \n",
"\ttpep_pickup_datetime TIMESTAMP WITHOUT TIME ZONE, \n",
"\ttpep_dropoff_datetime TIMESTAMP WITHOUT TIME ZONE, \n",
"\tpassenger_count FLOAT(53), \n",
"\ttrip_distance FLOAT(53), \n",
"\t\"RatecodeID\" FLOAT(53), \n",
"\tstore_and_fwd_flag TEXT, \n",
"\t\"PULocationID\" INTEGER, \n",
"\t\"DOLocationID\" INTEGER, \n",
"\tpayment_type BIGINT, \n",
"\tfare_amount FLOAT(53), \n",
"\textra FLOAT(53), \n",
"\tmta_tax FLOAT(53), \n",
"\ttip_amount FLOAT(53), \n",
"\ttolls_amount FLOAT(53), \n",
"\timprovement_surcharge FLOAT(53), \n",
"\ttotal_amount FLOAT(53), \n",
"\tcongestion_surcharge FLOAT(53), \n",
"\t\"Airport_fee\" FLOAT(53)\n",
")\n",
"\n",
"\n"
]
}
],
"source": [
"# Generate CREATE SQL statement from schema for validation\n",
"print(pd.io.sql.get_schema(df, name='yellow_taxi_data', con=engine))"
]
},
{
"cell_type": "markdown",
"id": "eca7f32d",
"metadata": {},
"source": [
"Datatypes for the table looks good! Since we used paraquet file the datasets seem to have been preserved. You may have to convert some datatypes so it is always good to do this check."
]
},
{
"cell_type": "markdown",
"id": "51a751ed",
"metadata": {},
"source": [
"## Finally inserting data\n",
"\n",
"There are 2,846,722 rows in our dataset. We are going to use the ```parquet_file.iter_batches()``` function to create batches of 100,000, convert them into pandas and then load it into the postgres database."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e20cec73",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-03T23:49:28.768786Z",
"start_time": "2023-12-03T23:49:28.689732Z"
},
"scrolled": true
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>VendorID</th>\n",
" <th>tpep_pickup_datetime</th>\n",
" <th>tpep_dropoff_datetime</th>\n",
" <th>passenger_count</th>\n",
" <th>trip_distance</th>\n",
" <th>RatecodeID</th>\n",
" <th>store_and_fwd_flag</th>\n",
" <th>PULocationID</th>\n",
" <th>DOLocationID</th>\n",
" <th>payment_type</th>\n",
" <th>fare_amount</th>\n",
" <th>extra</th>\n",
" <th>mta_tax</th>\n",
" <th>tip_amount</th>\n",
" <th>tolls_amount</th>\n",
" <th>improvement_surcharge</th>\n",
" <th>total_amount</th>\n",
" <th>congestion_surcharge</th>\n",
" <th>Airport_fee</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>2023-09-01 00:15:37</td>\n",
" <td>2023-09-01 00:20:21</td>\n",
" <td>1</td>\n",
" <td>0.80</td>\n",
" <td>1</td>\n",
" <td>N</td>\n",
" <td>163</td>\n",
" <td>230</td>\n",
" <td>2</td>\n",
" <td>6.5</td>\n",
" <td>3.5</td>\n",
" <td>0.5</td>\n",
" <td>0.00</td>\n",
" <td>0.00</td>\n",
" <td>1.0</td>\n",
" <td>11.50</td>\n",
" <td>2.5</td>\n",
" <td>0.00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>2023-09-01 00:18:40</td>\n",
" <td>2023-09-01 00:30:28</td>\n",
" <td>2</td>\n",
" <td>2.34</td>\n",
" <td>1</td>\n",
" <td>N</td>\n",
" <td>236</td>\n",
" <td>233</td>\n",
" <td>1</td>\n",
" <td>14.2</td>\n",
" <td>1.0</td>\n",
" <td>0.5</td>\n",
" <td>2.00</td>\n",
" <td>0.00</td>\n",
" <td>1.0</td>\n",
" <td>21.20</td>\n",
" <td>2.5</td>\n",
" <td>0.00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2</td>\n",
" <td>2023-09-01 00:35:01</td>\n",
" <td>2023-09-01 00:39:04</td>\n",
" <td>1</td>\n",
" <td>1.62</td>\n",
" <td>1</td>\n",
" <td>N</td>\n",
" <td>162</td>\n",
" <td>236</td>\n",
" <td>1</td>\n",
" <td>8.6</td>\n",
" <td>1.0</td>\n",
" <td>0.5</td>\n",
" <td>2.00</td>\n",
" <td>0.00</td>\n",
" <td>1.0</td>\n",
" <td>15.60</td>\n",
" <td>2.5</td>\n",
" <td>0.00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2</td>\n",
" <td>2023-09-01 00:45:45</td>\n",
" <td>2023-09-01 00:47:37</td>\n",
" <td>1</td>\n",
" <td>0.74</td>\n",
" <td>1</td>\n",
" <td>N</td>\n",
" <td>141</td>\n",
" <td>229</td>\n",
" <td>1</td>\n",
" <td>5.1</td>\n",
" <td>1.0</td>\n",
" <td>0.5</td>\n",
" <td>1.00</td>\n",
" <td>0.00</td>\n",
" <td>1.0</td>\n",
" <td>11.10</td>\n",
" <td>2.5</td>\n",
" <td>0.00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2</td>\n",
" <td>2023-09-01 00:01:23</td>\n",
" <td>2023-09-01 00:38:05</td>\n",
" <td>1</td>\n",
" <td>9.85</td>\n",
" <td>1</td>\n",
" <td>N</td>\n",
" <td>138</td>\n",
" <td>230</td>\n",
" <td>1</td>\n",
" <td>45.0</td>\n",
" <td>6.0</td>\n",
" <td>0.5</td>\n",
" <td>17.02</td>\n",
" <td>0.00</td>\n",
" <td>1.0</td>\n",
" <td>73.77</td>\n",
" <td>2.5</td>\n",
" <td>1.75</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>99995</th>\n",
" <td>2</td>\n",
" <td>2023-09-02 09:55:17</td>\n",
" <td>2023-09-02 10:01:45</td>\n",
" <td>2</td>\n",
" <td>1.48</td>\n",
" <td>1</td>\n",
" <td>N</td>\n",
" <td>163</td>\n",
" <td>164</td>\n",
" <td>1</td>\n",
" <td>9.3</td>\n",
" <td>0.0</td>\n",
" <td>0.5</td>\n",
" <td>2.66</td>\n",
" <td>0.00</td>\n",
" <td>1.0</td>\n",
" <td>15.96</td>\n",
" <td>2.5</td>\n",
" <td>0.00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>99996</th>\n",
" <td>2</td>\n",
" <td>2023-09-02 09:25:34</td>\n",
" <td>2023-09-02 09:55:20</td>\n",
" <td>3</td>\n",
" <td>17.49</td>\n",
" <td>2</td>\n",
" <td>N</td>\n",
" <td>132</td>\n",
" <td>164</td>\n",
" <td>1</td>\n",
" <td>70.0</td>\n",
" <td>0.0</td>\n",
" <td>0.5</td>\n",
" <td>24.28</td>\n",
" <td>6.94</td>\n",
" <td>1.0</td>\n",
" <td>106.97</td>\n",
" <td>2.5</td>\n",
" <td>1.75</td>\n",
" </tr>\n",
" <tr>\n",
" <th>99997</th>\n",
" <td>2</td>\n",
" <td>2023-09-02 09:57:55</td>\n",
" <td>2023-09-02 10:04:52</td>\n",
" <td>1</td>\n",
" <td>1.73</td>\n",
" <td>1</td>\n",
" <td>N</td>\n",
" <td>164</td>\n",
" <td>249</td>\n",
" <td>1</td>\n",
" <td>10.0</td>\n",
" <td>0.0</td>\n",
" <td>0.5</td>\n",
" <td>2.80</td>\n",
" <td>0.00</td>\n",
" <td>1.0</td>\n",
" <td>16.80</td>\n",
" <td>2.5</td>\n",
" <td>0.00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>99998</th>\n",
" <td>2</td>\n",
" <td>2023-09-02 09:35:02</td>\n",
" <td>2023-09-02 09:43:28</td>\n",
" <td>1</td>\n",
" <td>1.32</td>\n",
" <td>1</td>\n",
" <td>N</td>\n",
" <td>113</td>\n",
" <td>170</td>\n",
" <td>1</td>\n",
" <td>10.0</td>\n",
" <td>0.0</td>\n",
" <td>0.5</td>\n",
" <td>4.20</td>\n",
" <td>0.00</td>\n",
" <td>1.0</td>\n",
" <td>18.20</td>\n",
" <td>2.5</td>\n",
" <td>0.00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>99999</th>\n",
" <td>2</td>\n",
" <td>2023-09-02 09:46:09</td>\n",
" <td>2023-09-02 10:03:58</td>\n",
" <td>1</td>\n",
" <td>8.79</td>\n",
" <td>1</td>\n",
" <td>N</td>\n",
" <td>138</td>\n",
" <td>170</td>\n",
" <td>1</td>\n",
" <td>35.9</td>\n",
" <td>5.0</td>\n",
" <td>0.5</td>\n",
" <td>10.37</td>\n",
" <td>6.94</td>\n",
" <td>1.0</td>\n",
" <td>63.96</td>\n",
" <td>2.5</td>\n",
" <td>1.75</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>100000 rows × 19 columns</p>\n",
"</div>"
],
"text/plain": [
" VendorID tpep_pickup_datetime tpep_dropoff_datetime passenger_count \\\n",
"0 1 2023-09-01 00:15:37 2023-09-01 00:20:21 1 \n",
"1 2 2023-09-01 00:18:40 2023-09-01 00:30:28 2 \n",
"2 2 2023-09-01 00:35:01 2023-09-01 00:39:04 1 \n",
"3 2 2023-09-01 00:45:45 2023-09-01 00:47:37 1 \n",
"4 2 2023-09-01 00:01:23 2023-09-01 00:38:05 1 \n",
"... ... ... ... ... \n",
"99995 2 2023-09-02 09:55:17 2023-09-02 10:01:45 2 \n",
"99996 2 2023-09-02 09:25:34 2023-09-02 09:55:20 3 \n",
"99997 2 2023-09-02 09:57:55 2023-09-02 10:04:52 1 \n",
"99998 2 2023-09-02 09:35:02 2023-09-02 09:43:28 1 \n",
"99999 2 2023-09-02 09:46:09 2023-09-02 10:03:58 1 \n",
"\n",
" trip_distance RatecodeID store_and_fwd_flag PULocationID \\\n",
"0 0.80 1 N 163 \n",
"1 2.34 1 N 236 \n",
"2 1.62 1 N 162 \n",
"3 0.74 1 N 141 \n",
"4 9.85 1 N 138 \n",
"... ... ... ... ... \n",
"99995 1.48 1 N 163 \n",
"99996 17.49 2 N 132 \n",
"99997 1.73 1 N 164 \n",
"99998 1.32 1 N 113 \n",
"99999 8.79 1 N 138 \n",
"\n",
" DOLocationID payment_type fare_amount extra mta_tax tip_amount \\\n",
"0 230 2 6.5 3.5 0.5 0.00 \n",
"1 233 1 14.2 1.0 0.5 2.00 \n",
"2 236 1 8.6 1.0 0.5 2.00 \n",
"3 229 1 5.1 1.0 0.5 1.00 \n",
"4 230 1 45.0 6.0 0.5 17.02 \n",
"... ... ... ... ... ... ... \n",
"99995 164 1 9.3 0.0 0.5 2.66 \n",
"99996 164 1 70.0 0.0 0.5 24.28 \n",
"99997 249 1 10.0 0.0 0.5 2.80 \n",
"99998 170 1 10.0 0.0 0.5 4.20 \n",
"99999 170 1 35.9 5.0 0.5 10.37 \n",
"\n",
" tolls_amount improvement_surcharge total_amount \\\n",
"0 0.00 1.0 11.50 \n",
"1 0.00 1.0 21.20 \n",
"2 0.00 1.0 15.60 \n",
"3 0.00 1.0 11.10 \n",
"4 0.00 1.0 73.77 \n",
"... ... ... ... \n",
"99995 0.00 1.0 15.96 \n",
"99996 6.94 1.0 106.97 \n",
"99997 0.00 1.0 16.80 \n",
"99998 0.00 1.0 18.20 \n",
"99999 6.94 1.0 63.96 \n",
"\n",
" congestion_surcharge Airport_fee \n",
"0 2.5 0.00 \n",
"1 2.5 0.00 \n",
"2 2.5 0.00 \n",
"3 2.5 0.00 \n",
"4 2.5 1.75 \n",
"... ... ... \n",
"99995 2.5 0.00 \n",
"99996 2.5 1.75 \n",
"99997 2.5 0.00 \n",
"99998 2.5 0.00 \n",
"99999 2.5 1.75 \n",
"\n",
"[100000 rows x 19 columns]"
]
},
"execution_count": 66,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#This part is for testing\n",
"\n",
"\n",
"# Creating batches of 100,000 for the paraquet file\n",
"batches_iter = file.iter_batches(batch_size=100000)\n",
"batches_iter\n",
"\n",
"# Take the first batch for testing\n",
"df = next(batches_iter).to_pandas()\n",
"df\n",
"\n",
"# Creating just the table in postgres\n",
"#df.head(0).to_sql(name='ny_taxi_data',con=engine, if_exists='replace')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7fdda025",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-04T00:08:07.651559Z",
"start_time": "2023-12-04T00:02:35.940526Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"inserting batch 1...\n",
"inserted! time taken 12.916 seconds.\n",
"\n",
"inserting batch 2...\n",
"inserted! time taken 11.782 seconds.\n",
"\n",
"inserting batch 3...\n",
"inserted! time taken 11.854 seconds.\n",
"\n",
"inserting batch 4...\n",
"inserted! time taken 11.753 seconds.\n",
"\n",
"inserting batch 5...\n",
"inserted! time taken 12.034 seconds.\n",
"\n",
"inserting batch 6...\n",
"inserted! time taken 11.742 seconds.\n",
"\n",
"inserting batch 7...\n",
"inserted! time taken 12.351 seconds.\n",
"\n",
"inserting batch 8...\n",
"inserted! time taken 11.052 seconds.\n",
"\n",
"inserting batch 9...\n",
"inserted! time taken 12.167 seconds.\n",
"\n",
"inserting batch 10...\n",
"inserted! time taken 12.335 seconds.\n",
"\n",
"inserting batch 11...\n",
"inserted! time taken 11.375 seconds.\n",
"\n",
"inserting batch 12...\n",
"inserted! time taken 10.937 seconds.\n",
"\n",
"inserting batch 13...\n",
"inserted! time taken 12.208 seconds.\n",
"\n",
"inserting batch 14...\n",
"inserted! time taken 11.542 seconds.\n",
"\n",
"inserting batch 15...\n",
"inserted! time taken 11.460 seconds.\n",
"\n",
"inserting batch 16...\n",
"inserted! time taken 11.868 seconds.\n",
"\n",
"inserting batch 17...\n",
"inserted! time taken 11.162 seconds.\n",
"\n",
"inserting batch 18...\n",
"inserted! time taken 11.774 seconds.\n",
"\n",
"inserting batch 19...\n",
"inserted! time taken 11.772 seconds.\n",
"\n",
"inserting batch 20...\n",
"inserted! time taken 10.971 seconds.\n",
"\n",
"inserting batch 21...\n",
"inserted! time taken 11.483 seconds.\n",
"\n",
"inserting batch 22...\n",
"inserted! time taken 11.718 seconds.\n",
"\n",
"inserting batch 23...\n",
"inserted! time taken 11.628 seconds.\n",
"\n",
"inserting batch 24...\n",
"inserted! time taken 11.622 seconds.\n",
"\n",
"inserting batch 25...\n",
"inserted! time taken 11.236 seconds.\n",
"\n",
"inserting batch 26...\n",
"inserted! time taken 11.258 seconds.\n",
"\n",
"inserting batch 27...\n",
"inserted! time taken 11.746 seconds.\n",
"\n",
"inserting batch 28...\n",
"inserted! time taken 10.031 seconds.\n",
"\n",
"inserting batch 29...\n",
"inserted! time taken 5.077 seconds.\n",
"\n",
"Completed! Total time taken was 331.674 seconds for 29 batches.\n"
]
}
],
"source": [
"# Insert values into the table \n",
"t_start = time()\n",
"count = 0\n",
"for batch in file.iter_batches(batch_size=100000):\n",
" count+=1\n",
" batch_df = batch.to_pandas()\n",
" print(f'inserting batch {count}...')\n",
" b_start = time()\n",
" \n",
" batch_df.to_sql(name='ny_taxi_data',con=engine, if_exists='append')\n",
" b_end = time()\n",
" print(f'inserted! time taken {b_end-b_start:10.3f} seconds.\\n')\n",
" \n",
"t_end = time() \n",
"print(f'Completed! Total time taken was {t_end-t_start:10.3f} seconds for {count} batches.') "
]
},
{
"cell_type": "markdown",
"id": "a7c102be",
"metadata": {},
"source": [
"## Extra bit\n",
"\n",
"While trying to do the SQL Refresher, there was a need to add a lookup zones table but the file is in ```.csv``` format. \n",
"\n",
"Let's code to handle both ```.csv``` and ```.paraquet``` files!"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a643d171",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-05T20:59:29.236458Z",
"start_time": "2023-12-05T20:59:28.551221Z"
}
},
"outputs": [],
"source": [
"from time import time\n",
"import pandas as pd \n",
"import pyarrow.parquet as pq\n",
"from sqlalchemy import create_engine"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "62c9040a",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-05T21:18:11.346552Z",
"start_time": "2023-12-05T21:18:11.337475Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"'yellow_tripdata_2023-09.parquet'"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"url = 'https://d37ci6vzurychx.cloudfront.net/misc/taxi+_zone_lookup.csv'\n",
"url = 'https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2023-09.parquet'\n",
"\n",
"file_name = url.rsplit('/', 1)[-1].strip()\n",
"file_name"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e495fa96",
"metadata": {
"ExecuteTime": {
"end_time": "2023-12-05T21:18:33.001561Z",
"start_time": "2023-12-05T21:18:32.844872Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"oh yea\n"
]
}
],
"source": [
"if '.csv' in file_name:\n",
" print('yay') \n",
" df = pd.read_csv(file_name, nrows=10)\n",
" df_iter = pd.read_csv(file_name, iterator=True, chunksize=100000)\n",
"elif '.parquet' in file_name:\n",
" print('oh yea')\n",
" file = pq.ParquetFile(file_name)\n",
" df = next(file.iter_batches(batch_size=10)).to_pandas()\n",
" df_iter = file.iter_batches(batch_size=100000)\n",
"else: \n",
" print('Error. Only .csv or .parquet files allowed.')\n",
" sys.exit() "
]
},
{
"cell_type": "markdown",
"id": "7556748f",
"metadata": {},
"source": [
"This code is a rough code and seems to be working. The cleaned up version will be in `data-loading-parquet.py` file."
]
}
],
"metadata": {
"hide_input": false,
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.5"
},
"varInspector": {
"cols": {
"lenName": 16,
"lenType": 16,
"lenVar": 40
},
"kernels_config": {
"python": {
"delete_cmd_postfix": "",
"delete_cmd_prefix": "del ",
"library": "var_list.py",
"varRefreshCmd": "print(var_dic_list())"
},
"r": {
"delete_cmd_postfix": ") ",
"delete_cmd_prefix": "rm(",
"library": "var_list.r",
"varRefreshCmd": "cat(var_dic_list()) "
}
},
"types_to_exclude": [
"module",
"function",
"builtin_function_or_method",
"instance",
"_Feature"
],
"window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 5
}

86
01-docker-terraform/2_docker_sql/data-loading-parquet.py
View File

@ -1,86 +0,0 @@
#Cleaned up version of data-loading.ipynb
import argparse, os, sys
from time import time
import pandas as pd
import pyarrow.parquet as pq
from sqlalchemy import create_engine
def main(params):
user = params.user
password = params.password
host = params.host
port = params.port
db = params.db
tb = params.tb
url = params.url
# Get the name of the file from url
file_name = url.rsplit('/', 1)[-1].strip()
print(f'Downloading {file_name} ...')
# Download file from url
os.system(f'curl {url.strip()} -o {file_name}')
print('\n')
# Create SQL engine
engine = create_engine(f'postgresql://{user}:{password}@{host}:{port}/{db}')
# Read file based on csv or parquet
if '.csv' in file_name:
df = pd.read_csv(file_name, nrows=10)
df_iter = pd.read_csv(file_name, iterator=True, chunksize=100000)
elif '.parquet' in file_name:
file = pq.ParquetFile(file_name)
df = next(file.iter_batches(batch_size=10)).to_pandas()
df_iter = file.iter_batches(batch_size=100000)
else:
print('Error. Only .csv or .parquet files allowed.')
sys.exit()
# Create the table
df.head(0).to_sql(name=tb, con=engine, if_exists='replace')
# Insert values
t_start = time()
count = 0
for batch in df_iter:
count+=1
if '.parquet' in file_name:
batch_df = batch.to_pandas()
else:
batch_df = batch
print(f'inserting batch {count}...')
b_start = time()
batch_df.to_sql(name=tb, con=engine, if_exists='append')
b_end = time()
print(f'inserted! time taken {b_end-b_start:10.3f} seconds.\n')
t_end = time()
print(f'Completed! Total time taken was {t_end-t_start:10.3f} seconds for {count} batches.')
if __name__ == '__main__':
#Parsing arguments
parser = argparse.ArgumentParser(description='Loading data from .paraquet file link to a Postgres datebase.')
parser.add_argument('--user', help='Username for Postgres.')
parser.add_argument('--password', help='Password to the username for Postgres.')
parser.add_argument('--host', help='Hostname for Postgres.')
parser.add_argument('--port', help='Port for Postgres connection.')
parser.add_argument('--db', help='Databse name for Postgres')
parser.add_argument('--tb', help='Destination table name for Postgres.')
parser.add_argument('--url', help='URL for .paraquet file.')
args = parser.parse_args()
main(args)

211
01-docker-terraform/README.md
View File

@ -1,211 +0,0 @@
# Introduction
* [![](https://markdown-videos-api.jorgenkh.no/youtube/AtRhA-NfS24)](https://www.youtube.com/watch?v=AtRhA-NfS24&list=PL3MmuxUbc_hKihpnNQ9qtTmWYy26bPrSb&index=3)
* [Slides](https://www.slideshare.net/AlexeyGrigorev/data-engineering-zoomcamp-introduction)
* Overview of [Architecture](https://github.com/DataTalksClub/data-engineering-zoomcamp#overview), [Technologies](https://github.com/DataTalksClub/data-engineering-zoomcamp#technologies) & [Pre-Requisites](https://github.com/DataTalksClub/data-engineering-zoomcamp#prerequisites)
We suggest watching videos in the same order as in this document.
The last video (setting up the environment) is optional, but you can check it earlier
if you have troubles setting up the environment and following along with the videos.
# Docker + Postgres
[Code](2_docker_sql)
## :movie_camera: Introduction to Docker
[![](https://markdown-videos-api.jorgenkh.no/youtube/EYNwNlOrpr0)](https://youtu.be/EYNwNlOrpr0&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=4)
* Why do we need Docker
* Creating a simple "data pipeline" in Docker
## :movie_camera: Ingesting NY Taxi Data to Postgres
[![](https://markdown-videos-api.jorgenkh.no/youtube/2JM-ziJt0WI)](https://youtu.be/2JM-ziJt0WI&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=5)
* Running Postgres locally with Docker
* Using `pgcli` for connecting to the database
* Exploring the NY Taxi dataset
* Ingesting the data into the database
> [!TIP]
>if you have problems with `pgcli`, check this video for an alternative way to connect to your database in jupyter notebook and pandas.
>
> [![](https://markdown-videos-api.jorgenkh.no/youtube/3IkfkTwqHx4)](https://youtu.be/3IkfkTwqHx4&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=6)
## :movie_camera: Connecting pgAdmin and Postgres
[![](https://markdown-videos-api.jorgenkh.no/youtube/hCAIVe9N0ow)](https://youtu.be/hCAIVe9N0ow&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=7)
* The pgAdmin tool
* Docker networks
> [!IMPORTANT]
>The UI for PgAdmin 4 has changed, please follow the below steps for creating a server:
>
>* After login to PgAdmin, right click Servers in the left sidebar.
>* Click on Register.
>* Click on Server.
>* The remaining steps to create a server are the same as in the videos.
## :movie_camera: Putting the ingestion script into Docker
[![](https://markdown-videos-api.jorgenkh.no/youtube/B1WwATwf-vY)](https://youtu.be/B1WwATwf-vY&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=8)
* Converting the Jupyter notebook to a Python script
* Parametrizing the script with argparse
* Dockerizing the ingestion script
## :movie_camera: Running Postgres and pgAdmin with Docker-Compose
[![](https://markdown-videos-api.jorgenkh.no/youtube/hKI6PkPhpa0)](https://youtu.be/hKI6PkPhpa0&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=9)
* Why do we need Docker-compose
* Docker-compose YAML file
* Running multiple containers with `docker-compose up`
## :movie_camera: SQL refresher
[![](https://markdown-videos-api.jorgenkh.no/youtube/QEcps_iskgg)](https://youtu.be/QEcps_iskgg&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=10)
* Adding the Zones table
* Inner joins
* Basic data quality checks
* Left, Right and Outer joins
* Group by
## :movie_camera: Optional: Docker Networking and Port Mapping
> [!TIP]
> Optional: If you have some problems with docker networking, check **Port Mapping and Networks in Docker video**.
[![](https://markdown-videos-api.jorgenkh.no/youtube/tOr4hTsHOzU)](https://youtu.be/tOr4hTsHOzU&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=5)
* Docker networks
* Port forwarding to the host environment
* Communicating between containers in the network
* `.dockerignore` file
## :movie_camera: Optional: Walk-Through on WSL
> [!TIP]
> Optional: If you are willing to do the steps from "Ingesting NY Taxi Data to Postgres" till "Running Postgres and pgAdmin with Docker-Compose" with Windows Subsystem Linux please check **Docker Module Walk-Through on WSL**.
[![](https://markdown-videos-api.jorgenkh.no/youtube/Mv4zFm2AwzQ)](https://youtu.be/Mv4zFm2AwzQ&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=33)
# GCP
## :movie_camera: Introduction to GCP (Google Cloud Platform)
[![](https://markdown-videos-api.jorgenkh.no/youtube/18jIzE41fJ4)](https://youtu.be/18jIzE41fJ4&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=3)
# Terraform
[Code](1_terraform_gcp)
## :movie_camera: Introduction Terraform: Concepts and Overview, a primer
[![](https://markdown-videos-api.jorgenkh.no/youtube/s2bOYDCKl_M)](https://youtu.be/s2bOYDCKl_M&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=11)
* [Companion Notes](1_terraform_gcp)
## :movie_camera: Terraform Basics: Simple one file Terraform Deployment
[![](https://markdown-videos-api.jorgenkh.no/youtube/Y2ux7gq3Z0o)](https://youtu.be/Y2ux7gq3Z0o&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=12)
* [Companion Notes](1_terraform_gcp)
## :movie_camera: Deployment with a Variables File
[![](https://markdown-videos-api.jorgenkh.no/youtube/PBi0hHjLftk)](https://youtu.be/PBi0hHjLftk&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=13)
* [Companion Notes](1_terraform_gcp)
## Configuring terraform and GCP SDK on Windows
* [Instructions](1_terraform_gcp/windows.md)
# Environment setup
For the course you'll need:
* Python 3 (e.g. installed with Anaconda)
* Google Cloud SDK
* Docker with docker-compose
* Terraform
* Git account
> [!NOTE]
>If you have problems setting up the environment, you can check these videos.
>
>If you already have a working coding environment on local machine, these are optional. And only need to select one method. But if you have time to learn it now, these would be helpful if the local environment suddenly do not work one day.
## :movie_camera: GCP Cloud VM
### Setting up the environment on cloud VM
[![](https://markdown-videos-api.jorgenkh.no/youtube/ae-CV2KfoN0)](https://youtu.be/ae-CV2KfoN0&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=14)
* Generating SSH keys
* Creating a virtual machine on GCP
* Connecting to the VM with SSH
* Installing Anaconda
* Installing Docker
* Creating SSH `config` file
* Accessing the remote machine with VS Code and SSH remote
* Installing docker-compose
* Installing pgcli
* Port-forwarding with VS code: connecting to pgAdmin and Jupyter from the local computer
* Installing Terraform
* Using `sftp` for putting the credentials to the remote machine
* Shutting down and removing the instance
## :movie_camera: GitHub Codespaces
### Preparing the environment with GitHub Codespaces
[![](https://markdown-videos-api.jorgenkh.no/youtube/XOSUt8Ih3zA)](https://youtu.be/XOSUt8Ih3zA&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=15)
# Homework
* [Homework](../cohorts/2024/01-docker-terraform/homework.md)
# Community notes
Did you take notes? You can share them here
* [Notes from Alvaro Navas](https://github.com/ziritrion/dataeng-zoomcamp/blob/main/notes/1_intro.md)
* [Notes from Abd](https://itnadigital.notion.site/Week-1-Introduction-f18de7e69eb4453594175d0b1334b2f4)
* [Notes from Aaron](https://github.com/ABZ-Aaron/DataEngineerZoomCamp/blob/master/week_1_basics_n_setup/README.md)
* [Notes from Faisal](https://github.com/FaisalMohd/data-engineering-zoomcamp/blob/main/week_1_basics_n_setup/Notes/DE%20Zoomcamp%20Week-1.pdf)
* [Michael Harty's Notes](https://github.com/mharty3/data_engineering_zoomcamp_2022/tree/main/week01)
* [Blog post from Isaac Kargar](https://kargarisaac.github.io/blog/data%20engineering/jupyter/2022/01/18/data-engineering-w1.html)
* [Handwritten Notes By Mahmoud Zaher](https://github.com/zaherweb/DataEngineering/blob/master/week%201.pdf)
* [Notes from Candace Williams](https://teacherc.github.io/data-engineering/2023/01/18/zoomcamp1.html)
* [Notes from Marcos Torregrosa](https://www.n4gash.com/2023/data-engineering-zoomcamp-semana-1/)
* [Notes from Vincenzo Galante](https://binchentso.notion.site/Data-Talks-Club-Data-Engineering-Zoomcamp-8699af8e7ff94ec49e6f9bdec8eb69fd)
* [Notes from Victor Padilha](https://github.com/padilha/de-zoomcamp/tree/master/week1)
* [Notes from froukje](https://github.com/froukje/de-zoomcamp/blob/main/week_1_basics_n_setup/notes/notes_week_01.md)
* [Notes from adamiaonr](https://github.com/adamiaonr/data-engineering-zoomcamp/blob/main/week_1_basics_n_setup/2_docker_sql/NOTES.md)
* [Notes from Xia He-Bleinagel](https://xiahe-bleinagel.com/2023/01/week-1-data-engineering-zoomcamp-notes/)
* [Notes from Balaji](https://github.com/Balajirvp/DE-Zoomcamp/blob/main/Week%201/Detailed%20Week%201%20Notes.ipynb)
* [Notes from Erik](https://twitter.com/ehub96/status/1621351266281730049)
* [Notes by Alain Boisvert](https://github.com/boisalai/de-zoomcamp-2023/blob/main/week1.md)
* Notes on [Docker, Docker Compose, and setting up a proper Python environment](https://medium.com/@verazabeida/zoomcamp-2023-week-1-f4f94cb360ae), by Vera
* [Setting up the development environment on Google Virtual Machine](https://itsadityagupta.hashnode.dev/setting-up-the-development-environment-on-google-virtual-machine), blog post by Aditya Gupta
* [Notes from Zharko Cekovski](https://www.zharconsulting.com/contents/data/data-engineering-bootcamp-2024/week-1-postgres-docker-and-ingestion-scripts/)
* [2024 Module-01 Walkthough video by ellacharmed on youtube](https://youtu.be/VUZshlVAnk4)
* [2024 Companion Module Walkthough slides by ellacharmed](https://github.com/ellacharmed/data-engineering-zoomcamp/blob/ella2024/cohorts/2024/01-docker-terraform/walkthrough-01.pdf)
* [2024 Module-01 Environment setup video by ellacharmed on youtube](https://youtu.be/Zce_Hd37NGs)
* [Docker Notes by Linda](https://github.com/inner-outer-space/de-zoomcamp-2024/blob/main/1a-docker_sql/readme.md) • [Terraform Notes by Linda](https://github.com/inner-outer-space/de-zoomcamp-2024/blob/main/1b-terraform_gcp/readme.md)
* [Notes from Hammad Tariq](https://github.com/hamad-tariq/HammadTariq-ZoomCamp2024/blob/9c8b4908416eb8cade3d7ec220e7664c003e9b11/week_1_basics_n_setup/README.md)
* Add your notes above this line

191
02-workflow-orchestration/README.md
View File

@ -1,191 +0,0 @@
> [!NOTE]
>If you're looking for Airflow videos from the 2022 edition, check the [2022 cohort folder](../cohorts/2022/week_2_data_ingestion/).
>
>If you're looking for Prefect videos from the 2023 edition, check the [2023 cohort folder](../cohorts/2023/week_2_data_ingestion/).
# Week 2: Workflow Orchestration
Welcome to Week 2 of the Data Engineering Zoomcamp! 🚀😤 This week, we'll be covering workflow orchestration with Mage.
Mage is an open-source, hybrid framework for transforming and integrating data. ✨
This week, you'll learn how to use the Mage platform to author and share _magical_ data pipelines. This will all be covered in the course, but if you'd like to learn a bit more about Mage, check out our docs [here](https://docs.mage.ai/introduction/overview).
* [2.2.1 - 📯 Intro to Orchestration](#221----intro-to-orchestration)
* [2.2.2 - 🧙‍♂️ Intro to Mage](#222---%EF%B8%8F-intro-to-mage)
* [2.2.3 - 🐘 ETL: API to Postgres](#223----etl-api-to-postgres)
* [2.2.4 - 🤓 ETL: API to GCS](#224----etl-api-to-gcs)
* [2.2.5 - 🔍 ETL: GCS to BigQuery](#225----etl-gcs-to-bigquery)
* [2.2.6 - 👨‍💻 Parameterized Execution](#226----parameterized-execution)
* [2.2.7 - 🤖 Deployment (Optional)](#227----deployment-optional)
* [2.2.8 - 🗒️ Homework](#228---️-homework)
* [2.2.9 - 👣 Next Steps](#229----next-steps)
## 📕 Course Resources
### 2.2.1 - 📯 Intro to Orchestration
In this section, we'll cover the basics of workflow orchestration. We'll discuss what it is, why it's important, and how it can be used to build data pipelines.
Videos
- 2.2.1a - What is Orchestration?
[![](https://markdown-videos-api.jorgenkh.no/youtube/Li8-MWHhTbo)](https://youtu.be/Li8-MWHhTbo&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=17)
Resources
- [Slides](https://docs.google.com/presentation/d/17zSxG5Z-tidmgY-9l7Al1cPmz4Slh4VPK6o2sryFYvw/)
### 2.2.2 - 🧙‍♂️ Intro to Mage
In this section, we'll introduce the Mage platform. We'll cover what makes Mage different from other orchestrators, the fundamental concepts behind Mage, and how to get started. To cap it off, we'll spin Mage up via Docker 🐳 and run a simple pipeline.
Videos
- 2.2.2a - What is Mage?
[![](https://markdown-videos-api.jorgenkh.no/youtube/AicKRcK3pa4)](https://youtu.be/AicKRcK3pa4&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=18)
- 2.2.2b - Configuring Mage
[![](https://markdown-videos-api.jorgenkh.no/youtube/tNiV7Wp08XE)](https://youtu.be/tNiV7Wp08XE&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=19)
- 2.2.2c - A Simple Pipeline
[![](https://markdown-videos-api.jorgenkh.no/youtube/stI-gg4QBnI)](https://youtu.be/stI-gg4QBnI&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=20)
Resources
- [Getting Started Repo](https://github.com/mage-ai/mage-zoomcamp)
- [Slides](https://docs.google.com/presentation/d/1y_5p3sxr6Xh1RqE6N8o2280gUzAdiic2hPhYUUD6l88/)
### 2.2.3 - 🐘 ETL: API to Postgres
Hooray! Mage is up and running. Now, let's build a _real_ pipeline. In this section, we'll build a simple ETL pipeline that loads data from an API into a Postgres database. Our database will be built using Docker— it will be running locally, but it's the same as if it were running in the cloud.
Videos
- 2.2.3a - Configuring Postgres
[![](https://markdown-videos-api.jorgenkh.no/youtube/pmhI-ezd3BE)](https://youtu.be/pmhI-ezd3BE&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=21)
- 2.2.3b - Writing an ETL Pipeline : API to postgres
[![](https://markdown-videos-api.jorgenkh.no/youtube/Maidfe7oKLs)](https://youtu.be/Maidfe7oKLs&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=22)
### 2.2.4 - 🤓 ETL: API to GCS
Ok, so we've written data _locally_ to a database, but what about the cloud? In this tutorial, we'll walk through the process of using Mage to extract, transform, and load data from an API to Google Cloud Storage (GCS).
We'll cover both writing _partitioned_ and _unpartitioned_ data to GCS and discuss _why_ you might want to do one over the other. Many data teams start with extracting data from a source and writing it to a data lake _before_ loading it to a structured data source, like a database.
Videos
- 2.2.4a - Configuring GCP
[![](https://markdown-videos-api.jorgenkh.no/youtube/00LP360iYvE)](https://youtu.be/00LP360iYvE&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=23)
- 2.2.4b - Writing an ETL Pipeline : API to GCS
[![](https://markdown-videos-api.jorgenkh.no/youtube/w0XmcASRUnc)](https://youtu.be/w0XmcASRUnc&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=24)
Resources
- [DTC Zoomcamp GCP Setup](../01-docker-terraform/1_terraform_gcp/2_gcp_overview.md)
### 2.2.5 - 🔍 ETL: GCS to BigQuery
Now that we've written data to GCS, let's load it into BigQuery. In this section, we'll walk through the process of using Mage to load our data from GCS to BigQuery. This closely mirrors a very common data engineering workflow: loading data from a data lake into a data warehouse.
Videos
- 2.2.5a - Writing an ETL Pipeline : GCS to BigQuery
[![](https://markdown-videos-api.jorgenkh.no/youtube/JKp_uzM-XsM)](https://youtu.be/JKp_uzM-XsM&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=25)
### 2.2.6 - 👨‍💻 Parameterized Execution
By now you're familiar with building pipelines, but what about adding parameters? In this video, we'll discuss some built-in runtime variables that exist in Mage and show you how to define your own! We'll also cover how to use these variables to parameterize your pipelines. Finally, we'll talk about what it means to *backfill* a pipeline and how to do it in Mage.
Videos
- 2.2.6a - Parameterized Execution
[![](https://markdown-videos-api.jorgenkh.no/youtube/H0hWjWxB-rg)](https://youtu.be/H0hWjWxB-rg&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=26)
- 2.2.6b - Backfills
[![](https://markdown-videos-api.jorgenkh.no/youtube/ZoeC6Ag5gQc)](https://youtu.be/ZoeC6Ag5gQc&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=27)
Resources
- [Mage Variables Overview](https://docs.mage.ai/development/variables/overview)
- [Mage Runtime Variables](https://docs.mage.ai/getting-started/runtime-variable)
### 2.2.7 - 🤖 Deployment (Optional)
In this section, we'll cover deploying Mage using Terraform and Google Cloud. This section is optional— it's not *necessary* to learn Mage, but it might be helpful if you're interested in creating a fully deployed project. If you're using Mage in your final project, you'll need to deploy it to the cloud.
Videos
- 2.2.7a - Deployment Prerequisites
[![](https://markdown-videos-api.jorgenkh.no/youtube/zAwAX5sxqsg)](https://youtu.be/zAwAX5sxqsg&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=28)
- 2.2.7b - Google Cloud Permissions
[![](https://markdown-videos-api.jorgenkh.no/youtube/O_H7DCmq2rA)](https://youtu.be/O_H7DCmq2rA&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=29)
- 2.2.7c - Deploying to Google Cloud - Part 1
[![](https://markdown-videos-api.jorgenkh.no/youtube/9A872B5hb_0)](https://youtu.be/9A872B5hb_0&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=30)
- 2.2.7d - Deploying to Google Cloud - Part 2
[![](https://markdown-videos-api.jorgenkh.no/youtube/0YExsb2HgLI)](https://youtu.be/0YExsb2HgLI&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=31)
Resources
- [Installing Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli)
- [Installing `gcloud` CLI](https://cloud.google.com/sdk/docs/install)
- [Mage Terraform Templates](https://github.com/mage-ai/mage-ai-terraform-templates)
Additional Mage Guides
- [Terraform](https://docs.mage.ai/production/deploying-to-cloud/using-terraform)
- [Deploying to GCP with Terraform](https://docs.mage.ai/production/deploying-to-cloud/gcp/setup)
### 2.2.8 - 🗒️ Homework
We've prepared a short exercise to test you on what you've learned this week. You can find the homework [here](../cohorts/2024/02-workflow-orchestration/homework.md). This follows closely from the contents of the course and shouldn't take more than an hour or two to complete. 😄
### 2.2.9 - 👣 Next Steps
Congratulations! You've completed Week 2 of the Data Engineering Zoomcamp. We hope you've enjoyed learning about Mage and that you're excited to use it in your final project. If you have any questions, feel free to reach out to us on Slack. Be sure to check out our "Next Steps" video for some inspiration for the rest of your journey 😄.
Videos
- 2.2.9 - Next Steps
[![](https://markdown-videos-api.jorgenkh.no/youtube/uUtj7N0TleQ)](https://youtu.be/uUtj7N0TleQ&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=32)
Resources
- [Slides](https://docs.google.com/presentation/d/1yN-e22VNwezmPfKrZkgXQVrX5owDb285I2HxHWgmAEQ/edit#slide=id.g262fb0d2905_0_12)
### 📑 Additional Resources
- [Mage Docs](https://docs.mage.ai/)
- [Mage Guides](https://docs.mage.ai/guides)
- [Mage Slack](https://www.mage.ai/chat)
# Community notes
Did you take notes? You can share them here:
## 2024 notes
* [2024 Videos transcripts week 2](https://drive.google.com/drive/folders/1yxT0uMMYKa6YOxanh91wGqmQUMS7yYW7?usp=sharing) by Maria Fisher
* [Notes from Jonah Oliver](https://www.jonahboliver.com/blog/de-zc-w2)
* [Notes from Linda](https://github.com/inner-outer-space/de-zoomcamp-2024/blob/main/2-workflow-orchestration/readme.md)
* [Notes from Kirill](https://github.com/kirill505/data-engineering-zoomcamp/blob/main/02-workflow-orchestration/README.md)
* [Notes from Zharko](https://www.zharconsulting.com/contents/data/data-engineering-bootcamp-2024/week-2-ingesting-data-with-mage/)
* Add your notes above this line
## 2023 notes
See [here](../cohorts/2023/week_2_workflow_orchestration#community-notes)
## 2022 notes
See [here](../cohorts/2022/week_2_data_ingestion#community-notes)

80
03-data-warehouse/README.md
View File

@ -1,80 +0,0 @@
# Data Warehouse and BigQuery
- [Slides](https://docs.google.com/presentation/d/1a3ZoBAXFk8-EhUsd7rAZd-5p_HpltkzSeujjRGB2TAI/edit?usp=sharing)
- [Big Query basic SQL](big_query.sql)
# Videos
## Data Warehouse
- Data Warehouse and BigQuery
[![](https://markdown-videos-api.jorgenkh.no/youtube/jrHljAoD6nM)](https://youtu.be/jrHljAoD6nM&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=34)
## :movie_camera: Partitoning and clustering
- Partioning and Clustering
[![](https://markdown-videos-api.jorgenkh.no/youtube/-CqXf7vhhDs)](https://youtu.be/-CqXf7vhhDs&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=35)
- Partioning vs Clustering
[![](https://markdown-videos-api.jorgenkh.no/youtube/-CqXf7vhhDs)](https://youtu.be/-CqXf7vhhDs?si=p1sYQCAs8dAa7jIm&t=193&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=35)
## :movie_camera: Best practices
[![](https://markdown-videos-api.jorgenkh.no/youtube/k81mLJVX08w)](https://youtu.be/k81mLJVX08w&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=36)
## :movie_camera: Internals of BigQuery
[![](https://markdown-videos-api.jorgenkh.no/youtube/eduHi1inM4s)](https://youtu.be/eduHi1inM4s&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=37)
## Advanced topics
### :movie_camera: Machine Learning in Big Query
[![](https://markdown-videos-api.jorgenkh.no/youtube/B-WtpB0PuG4)](https://youtu.be/B-WtpB0PuG4&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=34)
* [SQL for ML in BigQuery](big_query_ml.sql)
**Important links**
- [BigQuery ML Tutorials](https://cloud.google.com/bigquery-ml/docs/tutorials)
- [BigQuery ML Reference Parameter](https://cloud.google.com/bigquery-ml/docs/analytics-reference-patterns)
- [Hyper Parameter tuning](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create-glm)
- [Feature preprocessing](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-preprocess-overview)
### :movie_camera: Deploying Machine Learning model from BigQuery
[![](https://markdown-videos-api.jorgenkh.no/youtube/BjARzEWaznU)](https://youtu.be/BjARzEWaznU&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=39)
- [Steps to extract and deploy model with docker](extract_model.md)
# Homework
* [2024 Homework](../cohorts/2024/03-data-warehouse/homework.md)
# Community notes
Did you take notes? You can share them here.
* [Notes by Alvaro Navas](https://github.com/ziritrion/dataeng-zoomcamp/blob/main/notes/3_data_warehouse.md)
* [Isaac Kargar's blog post](https://kargarisaac.github.io/blog/data%20engineering/jupyter/2022/01/30/data-engineering-w3.html)
* [Marcos Torregrosa's blog post](https://www.n4gash.com/2023/data-engineering-zoomcamp-semana-3/)
* [Notes by Victor Padilha](https://github.com/padilha/de-zoomcamp/tree/master/week3)
* [Notes from Xia He-Bleinagel](https://xiahe-bleinagel.com/2023/02/week-3-data-engineering-zoomcamp-notes-data-warehouse-and-bigquery/)
* [Bigger picture summary on Data Lakes, Data Warehouses, and tooling](https://medium.com/@verazabeida/zoomcamp-week-4-b8bde661bf98), by Vera
* [Notes by froukje](https://github.com/froukje/de-zoomcamp/blob/main/week_3_data_warehouse/notes/notes_week_03.md)
* [Notes by Alain Boisvert](https://github.com/boisalai/de-zoomcamp-2023/blob/main/week3.md)
* [Notes from Vincenzo Galante](https://binchentso.notion.site/Data-Talks-Club-Data-Engineering-Zoomcamp-8699af8e7ff94ec49e6f9bdec8eb69fd)
* [2024 videos transcript week3](https://drive.google.com/drive/folders/1quIiwWO-tJCruqvtlqe_Olw8nvYSmmDJ?usp=sharing) by Maria Fisher
* [Notes by Linda](https://github.com/inner-outer-space/de-zoomcamp-2024/blob/main/3a-data-warehouse/readme.md)
* [Jonah Oliver's blog post](https://www.jonahboliver.com/blog/de-zc-w3)
* [2024 - steps to send data from Mage to GCS + creating external table](https://drive.google.com/file/d/1GIi6xnS4070a8MUlIg-ozITt485_-ePB/view?usp=drive_link) by Maria Fisher
* [2024 - mage dataloader script to load the parquet files from a remote URL and push it to Google bucket as parquet file](https://github.com/amohan601/dataengineering-zoomcamp2024/blob/main/week_3_data_warehouse/mage_scripts/green_taxi_2022_v2.py) by Anju Mohan
* [2024 - steps to send data from Mage to GCS + creating external table](https://drive.google.com/file/d/1GIi6xnS4070a8MUlIg-ozITt485_-ePB/view?usp=drive_link) by Maria Fisher
* [Notes by HongWei](https://github.com/hwchua0209/data-engineering-zoomcamp-submission/blob/main/03-data-warehouse/README.md)
* Add your notes here (above this line)

140
04-analytics-engineering/README.md
View File

@ -1,140 +0,0 @@
# Week 4: Analytics Engineering
Goal: Transforming the data loaded in DWH into Analytical Views developing a [dbt project](taxi_rides_ny/README.md).
### Prerequisites
By this stage of the course you should have already:
- A running warehouse (BigQuery or postgres)
- A set of running pipelines ingesting the project dataset (week 3 completed)
- The following datasets ingested from the course [Datasets list](https://github.com/DataTalksClub/nyc-tlc-data/):
* Yellow taxi data - Years 2019 and 2020
* Green taxi data - Years 2019 and 2020
* fhv data - Year 2019.
> [!NOTE]
> * We have two quick hack to load that data quicker, follow [this video](https://www.youtube.com/watch?v=Mork172sK_c&list=PLaNLNpjZpzwgneiI-Gl8df8GCsPYp_6Bs) for option 1 or check instructions in [week3/extras](../03-data-warehouse/extras) for option 2
## Setting up your environment
> [!NOTE]
> the *cloud* setup is the preferred option.
>
> the *local* setup does not require a cloud database.
| Alternative A | Alternative B |
---|---|
| Setting up dbt for using BigQuery (cloud) | Setting up dbt for using Postgres locally |
|- Open a free developer dbt cloud account following [this link](https://www.getdbt.com/signup/)|- Open a free developer dbt cloud account following [this link](https://www.getdbt.com/signup/)<br><br> |
| - [Following these instructions to connect to your BigQuery instance]([https://docs.getdbt.com/docs/dbt-cloud/cloud-configuring-dbt-cloud/cloud-setting-up-bigquery-oauth](https://docs.getdbt.com/guides/bigquery?step=4)) | - follow the [official dbt documentation]([https://docs.getdbt.com/dbt-cli/installation](https://docs.getdbt.com/docs/core/installation-overview)) or <br>- follow the [dbt core with BigQuery on Docker](docker_setup/README.md) guide to setup dbt locally on docker or <br>- use a docker image from oficial [Install with Docker](https://docs.getdbt.com/docs/core/docker-install). |
|- More detailed instructions in [dbt_cloud_setup.md](dbt_cloud_setup.md) | - You will need to install the latest version with the BigQuery adapter (dbt-bigquery).|
| | - You will need to install the latest version with the postgres adapter (dbt-postgres).|
| | After local installation you will have to set up the connection to PG in the `profiles.yml`, you can find the templates [here](https://docs.getdbt.com/docs/core/connect-data-platform/postgres-setup) |
## Content
### Introduction to analytics engineering
* What is analytics engineering?
* ETL vs ELT
* Data modeling concepts (fact and dim tables)
[![](https://markdown-videos-api.jorgenkh.no/youtube/uF76d5EmdtU)](https://youtu.be/uF76d5EmdtU&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=40)
### What is dbt?
* Introduction to dbt
[![](https://markdown-videos-api.jorgenkh.no/youtube/4eCouvVOJUw)](https://www.youtube.com/watch?v=gsKuETFJr54&list=PLaNLNpjZpzwgneiI-Gl8df8GCsPYp_6Bs&index=5)
## Starting a dbt project
| Alternative A | Alternative B |
|-----------------------------|--------------------------------|
| Using BigQuery + dbt cloud | Using Postgres + dbt core (locally) |
| - Starting a new project with dbt init (dbt cloud and core)<br>- dbt cloud setup<br>- project.yml<br><br> | - Starting a new project with dbt init (dbt cloud and core)<br>- dbt core local setup<br>- profiles.yml<br>- project.yml |
| [![](https://markdown-videos-api.jorgenkh.no/youtube/iMxh6s_wL4Q)](https://www.youtube.com/watch?v=J0XCDyKiU64&list=PLaNLNpjZpzwgneiI-Gl8df8GCsPYp_6Bs&index=4) | [![](https://markdown-videos-api.jorgenkh.no/youtube/1HmL63e-vRs)](https://youtu.be/1HmL63e-vRs&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=43) |
### dbt models
* Anatomy of a dbt model: written code vs compiled Sources
* Materialisations: table, view, incremental, ephemeral
* Seeds, sources and ref
* Jinja and Macros
* Packages
* Variables
[![](https://markdown-videos-api.jorgenkh.no/youtube/UVI30Vxzd6c)](https://www.youtube.com/watch?v=ueVy2N54lyc&list=PLaNLNpjZpzwgneiI-Gl8df8GCsPYp_6Bs&index=3)
> [!NOTE]
> *This video is shown entirely on dbt cloud IDE but the same steps can be followed locally on the IDE of your choice*
> [!TIP]
>* If you recieve an error stating "Permission denied while globbing file pattern." when attempting to run `fact_trips.sql` this video may be helpful in resolving the issue
>
>[![](https://markdown-videos-api.jorgenkh.no/youtube/kL3ZVNL9Y4A)](https://youtu.be/kL3ZVNL9Y4A&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=34)
### Testing and documenting dbt models
* Tests
* Documentation
[![](https://markdown-videos-api.jorgenkh.no/youtube/UishFmq1hLM)](https://www.youtube.com/watch?v=2dNJXHFCHaY&list=PLaNLNpjZpzwgneiI-Gl8df8GCsPYp_6Bs&index=2)
>[!NOTE]
> *This video is shown entirely on dbt cloud IDE but the same steps can be followed locally on the IDE of your choice*
## Deployment
| Alternative A | Alternative B |
|-----------------------------|--------------------------------|
| Using BigQuery + dbt cloud | Using Postgres + dbt core (locally) |
| - Deployment: development environment vs production<br>- dbt cloud: scheduler, sources and hosted documentation | - Deployment: development environment vs production<br>- dbt cloud: scheduler, sources and hosted documentation |
| [![](https://markdown-videos-api.jorgenkh.no/youtube/rjf6yZNGX8I)](https://www.youtube.com/watch?v=V2m5C0n8Gro&list=PLaNLNpjZpzwgneiI-Gl8df8GCsPYp_6Bs&index=6) | [![](https://markdown-videos-api.jorgenkh.no/youtube/Cs9Od1pcrzM)](https://youtu.be/Cs9Od1pcrzM&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=47) |
## Visualising the transformed data
:movie_camera: Google data studio Video (Now renamed to Looker studio)
[![](https://markdown-videos-api.jorgenkh.no/youtube/39nLTs74A3E)](https://youtu.be/39nLTs74A3E&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=48)
:movie_camera: Metabase Video
[![](https://markdown-videos-api.jorgenkh.no/youtube/BnLkrA7a6gM)](https://youtu.be/BnLkrA7a6gM&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=49)
## Advanced concepts
* [Make a model Incremental](https://docs.getdbt.com/docs/building-a-dbt-project/building-models/configuring-incremental-models)
* [Use of tags](https://docs.getdbt.com/reference/resource-configs/tags)
* [Hooks](https://docs.getdbt.com/docs/building-a-dbt-project/hooks-operations)
* [Analysis](https://docs.getdbt.com/docs/building-a-dbt-project/analyses)
* [Snapshots](https://docs.getdbt.com/docs/building-a-dbt-project/snapshots)
* [Exposure](https://docs.getdbt.com/docs/building-a-dbt-project/exposures)
* [Metrics](https://docs.getdbt.com/docs/building-a-dbt-project/metrics)
## Community notes
Did you take notes? You can share them here.
* [Notes by Alvaro Navas](https://github.com/ziritrion/dataeng-zoomcamp/blob/main/notes/4_analytics.md)
* [Sandy's DE learning blog](https://learningdataengineering540969211.wordpress.com/2022/02/17/week-4-setting-up-dbt-cloud-with-bigquery/)
* [Notes by Victor Padilha](https://github.com/padilha/de-zoomcamp/tree/master/week4)
* [Marcos Torregrosa's blog (spanish)](https://www.n4gash.com/2023/data-engineering-zoomcamp-semana-4/)
* [Notes by froukje](https://github.com/froukje/de-zoomcamp/blob/main/week_4_analytics_engineering/notes/notes_week_04.md)
* [Notes by Alain Boisvert](https://github.com/boisalai/de-zoomcamp-2023/blob/main/week4.md)
* [Setting up Prefect with dbt by Vera](https://medium.com/@verazabeida/zoomcamp-week-5-5b6a9d53a3a0)
* [Blog by Xia He-Bleinagel](https://xiahe-bleinagel.com/2023/02/week-4-data-engineering-zoomcamp-notes-analytics-engineering-and-dbt/)
* [Setting up DBT with BigQuery by Tofag](https://medium.com/@fagbuyit/setting-up-your-dbt-cloud-dej-9-d18e5b7c96ba)
* [Blog post by Dewi Oktaviani](https://medium.com/@oktavianidewi/de-zoomcamp-2023-learning-week-4-analytics-engineering-with-dbt-53f781803d3e)
* [Notes from Vincenzo Galante](https://binchentso.notion.site/Data-Talks-Club-Data-Engineering-Zoomcamp-8699af8e7ff94ec49e6f9bdec8eb69fd)
* [Notes from Balaji](https://github.com/Balajirvp/DE-Zoomcamp/blob/main/Week%204/Data%20Engineering%20Zoomcamp%20Week%204.ipynb)
* [Notes by Linda](https://github.com/inner-outer-space/de-zoomcamp-2024/blob/main/4-analytics-engineering/readme.md)
* [2024 - Videos transcript week4](https://drive.google.com/drive/folders/1V2sHWOotPEMQTdMT4IMki1fbMPTn3jOP?usp=drive)
* [Blog Post](https://www.jonahboliver.com/blog/de-zc-w4) by Jonah Oliver
* Add your notes here (above this line)
## Useful links
- [Slides used in the videos](https://docs.google.com/presentation/d/1xSll_jv0T8JF4rYZvLHfkJXYqUjPtThA/edit?usp=sharing&ouid=114544032874539580154&rtpof=true&sd=true)
- [Visualizing data with Metabase course](https://www.metabase.com/learn/visualization/)
- [dbt free courses](https://courses.getdbt.com/collections)

5
04-analytics-engineering/taxi_rides_ny/.gitignore vendored
View File

@ -1,5 +0,0 @@
# you shouldn't commit these into source control
# these are the default directory names, adjust/add to fit your needs
target/
dbt_packages/
logs/

49
04-analytics-engineering/taxi_rides_ny/analyses/hack-load-data.sql
View File

@ -1,49 +0,0 @@
-- MAKE SURE YOU REPLACE taxi-rides-ny-339813-412521 WITH THE NAME OF YOUR DATASET!
-- When you run the query, only run 5 of the ALTER TABLE statements at one time (by highlighting only 5).
-- Otherwise BigQuery will say too many alterations to the table are being made.
CREATE TABLE `taxi-rides-ny-339813-412521.trips_data_all.green_tripdata` as
SELECT * FROM `bigquery-public-data.new_york_taxi_trips.tlc_green_trips_2019`;
CREATE TABLE `taxi-rides-ny-339813-412521.trips_data_all.yellow_tripdata` as
SELECT * FROM `bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2019`;
insert into `taxi-rides-ny-339813-412521.trips_data_all.green_tripdata`
SELECT * FROM `bigquery-public-data.new_york_taxi_trips.tlc_green_trips_2020` ;
insert into `taxi-rides-ny-339813-412521.trips_data_all.yellow_tripdata`
SELECT * FROM `bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2020`;
-- Fixes yellow table schema
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.yellow_tripdata`
RENAME COLUMN vendor_id TO VendorID;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.yellow_tripdata`
RENAME COLUMN pickup_datetime TO tpep_pickup_datetime;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.yellow_tripdata`
RENAME COLUMN dropoff_datetime TO tpep_dropoff_datetime;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.yellow_tripdata`
RENAME COLUMN rate_code TO RatecodeID;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.yellow_tripdata`
RENAME COLUMN imp_surcharge TO improvement_surcharge;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.yellow_tripdata`
RENAME COLUMN pickup_location_id TO PULocationID;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.yellow_tripdata`
RENAME COLUMN dropoff_location_id TO DOLocationID;
-- Fixes green table schema
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.green_tripdata`
RENAME COLUMN vendor_id TO VendorID;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.green_tripdata`
RENAME COLUMN pickup_datetime TO lpep_pickup_datetime;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.green_tripdata`
RENAME COLUMN dropoff_datetime TO lpep_dropoff_datetime;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.green_tripdata`
RENAME COLUMN rate_code TO RatecodeID;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.green_tripdata`
RENAME COLUMN imp_surcharge TO improvement_surcharge;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.green_tripdata`
RENAME COLUMN pickup_location_id TO PULocationID;
ALTER TABLE `taxi-rides-ny-339813-412521.trips_data_all.green_tripdata`
RENAME COLUMN dropoff_location_id TO DOLocationID;

129
04-analytics-engineering/taxi_rides_ny/models/core/schema.yml
View File

@ -1,129 +0,0 @@
version: 2
models:
- name: dim_zones
description: >
List of unique zones idefied by locationid.
Includes the service zone they correspond to (Green or yellow).
- name: dm_monthly_zone_revenue
description: >
Aggregated table of all taxi trips corresponding to both service zones (Green and yellow) per pickup zone, month and service.
The table contains monthly sums of the fare elements used to calculate the monthly revenue.
The table contains also monthly indicators like number of trips, and average trip distance.
columns:
- name: revenue_monthly_total_amount
description: Monthly sum of the the total_amount of the fare charged for the trip per pickup zone, month and service.
tests:
- not_null:
severity: error
- name: fact_trips
description: >
Taxi trips corresponding to both service zones (Green and yellow).
The table contains records where both pickup and dropoff locations are valid and known zones.
Each record corresponds to a trip uniquely identified by tripid.
columns:
- name: tripid
data_type: string
description: "unique identifier conformed by the combination of vendorid and pickyp time"
- name: vendorid
data_type: int64
description: ""
- name: service_type
data_type: string
description: ""
- name: ratecodeid
data_type: int64
description: ""
- name: pickup_locationid
data_type: int64
description: ""
- name: pickup_borough
data_type: string
description: ""
- name: pickup_zone
data_type: string
description: ""
- name: dropoff_locationid
data_type: int64
description: ""
- name: dropoff_borough
data_type: string
description: ""
- name: dropoff_zone
data_type: string
description: ""
- name: pickup_datetime
data_type: timestamp
description: ""
- name: dropoff_datetime
data_type: timestamp
description: ""
- name: store_and_fwd_flag
data_type: string
description: ""
- name: passenger_count
data_type: int64
description: ""
- name: trip_distance
data_type: numeric
description: ""
- name: trip_type
data_type: int64
description: ""
- name: fare_amount
data_type: numeric
description: ""
- name: extra
data_type: numeric
description: ""
- name: mta_tax
data_type: numeric
description: ""
- name: tip_amount
data_type: numeric
description: ""
- name: tolls_amount
data_type: numeric
description: ""
- name: ehail_fee
data_type: numeric
description: ""
- name: improvement_surcharge
data_type: numeric
description: ""
- name: total_amount
data_type: numeric
description: ""
- name: payment_type
data_type: int64
description: ""
- name: payment_type_description
data_type: string
description: ""

52
04-analytics-engineering/taxi_rides_ny/models/staging/stg_green_tripdata.sql
View File

@ -1,52 +0,0 @@
{{
config(
materialized='view'
)
}}
with tripdata as
(
select *,
row_number() over(partition by vendorid, lpep_pickup_datetime) as rn
from {{ source('staging','green_tripdata') }}
where vendorid is not null
)
select
-- identifiers
{{ dbt_utils.generate_surrogate_key(['vendorid', 'lpep_pickup_datetime']) }} as tripid,
{{ dbt.safe_cast("vendorid", api.Column.translate_type("integer")) }} as vendorid,
{{ dbt.safe_cast("ratecodeid", api.Column.translate_type("integer")) }} as ratecodeid,
{{ dbt.safe_cast("pulocationid", api.Column.translate_type("integer")) }} as pickup_locationid,
{{ dbt.safe_cast("dolocationid", api.Column.translate_type("integer")) }} as dropoff_locationid,
-- timestamps
cast(lpep_pickup_datetime as timestamp) as pickup_datetime,
cast(lpep_dropoff_datetime as timestamp) as dropoff_datetime,
-- trip info
store_and_fwd_flag,
{{ dbt.safe_cast("passenger_count", api.Column.translate_type("integer")) }} as passenger_count,
cast(trip_distance as numeric) as trip_distance,
{{ dbt.safe_cast("trip_type", api.Column.translate_type("integer")) }} as trip_type,
-- payment info
cast(fare_amount as numeric) as fare_amount,
cast(extra as numeric) as extra,
cast(mta_tax as numeric) as mta_tax,
cast(tip_amount as numeric) as tip_amount,
cast(tolls_amount as numeric) as tolls_amount,
cast(ehail_fee as numeric) as ehail_fee,
cast(improvement_surcharge as numeric) as improvement_surcharge,
cast(total_amount as numeric) as total_amount,
coalesce({{ dbt.safe_cast("payment_type", api.Column.translate_type("integer")) }},0) as payment_type,
{{ get_payment_type_description("payment_type") }} as payment_type_description
from tripdata
where rn = 1
-- dbt build --select <model_name> --vars '{'is_test_run': 'false'}'
{% if var('is_test_run', default=true) %}
limit 100
{% endif %}

6
04-analytics-engineering/taxi_rides_ny/package-lock.yml
View File

@ -1,6 +0,0 @@
packages:
- package: dbt-labs/dbt_utils
version: 1.1.1
- package: dbt-labs/codegen
version: 0.12.1
sha1_hash: d974113b0f072cce35300077208f38581075ab40

5
04-analytics-engineering/taxi_rides_ny/packages.yml
View File

@ -1,5 +0,0 @@
packages:
- package: dbt-labs/dbt_utils
version: 1.1.1
- package: dbt-labs/codegen
version: 0.12.1

122
05-batch/README.md
View File

@ -1,122 +0,0 @@
# Week 5: Batch Processing
## 5.1 Introduction
* :movie_camera: 5.1.1 Introduction to Batch Processing
[![](https://markdown-videos-api.jorgenkh.no/youtube/dcHe5Fl3MF8)](https://youtu.be/dcHe5Fl3MF8&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=51)
* :movie_camera: 5.1.2 Introduction to Spark
[![](https://markdown-videos-api.jorgenkh.no/youtube/FhaqbEOuQ8U)](https://youtu.be/FhaqbEOuQ8U&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=52)
## 5.2 Installation
Follow [these intructions](setup/) to install Spark:
* [Windows](setup/windows.md)
* [Linux](setup/linux.md)
* [MacOS](setup/macos.md)
And follow [this](setup/pyspark.md) to run PySpark in Jupyter
* :movie_camera: 5.2.1 (Optional) Installing Spark (Linux)
[![](https://markdown-videos-api.jorgenkh.no/youtube/hqUbB9c8sKg)](https://youtu.be/hqUbB9c8sKg&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=53)
Alternatively, if the setups above don't work, you can run Spark in Google Colab.
> [!NOTE]
> It's advisable to invest some time in setting things up locally rather than immediately jumping into this solution
* [Google Colab Instructions](https://medium.com/gitconnected/launch-spark-on-google-colab-and-connect-to-sparkui-342cad19b304)
* [Google Colab Starter Notebook](https://github.com/aaalexlit/medium_articles/blob/main/Spark_in_Colab.ipynb)
## 5.3 Spark SQL and DataFrames
* :movie_camera: 5.3.1 First Look at Spark/PySpark
[![](https://markdown-videos-api.jorgenkh.no/youtube/r_Sf6fCB40c)](https://youtu.be/r_Sf6fCB40c&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=54)
* :movie_camera: 5.3.2 Spark Dataframes
[![](https://markdown-videos-api.jorgenkh.no/youtube/ti3aC1m3rE8)](https://youtu.be/ti3aC1m3rE8&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=55)
* :movie_camera: 5.3.3 (Optional) Preparing Yellow and Green Taxi Data
[![](https://markdown-videos-api.jorgenkh.no/youtube/CI3P4tAtru4)](https://youtu.be/CI3P4tAtru4&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=56)
Script to prepare the Dataset [download_data.sh](code/download_data.sh)
> [!NOTE]
> The other way to infer the schema (apart from pandas) for the csv files, is to set the `inferSchema` option to `true` while reading the files in Spark.
* :movie_camera: 5.3.4 SQL with Spark
[![](https://markdown-videos-api.jorgenkh.no/youtube/uAlp2VuZZPY)](https://youtu.be/uAlp2VuZZPY&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=57)
## 5.4 Spark Internals
* :movie_camera: 5.4.1 Anatomy of a Spark Cluster
[![](https://markdown-videos-api.jorgenkh.no/youtube/68CipcZt7ZA)](https://youtu.be/68CipcZt7ZA&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=58)
* :movie_camera: 5.4.2 GroupBy in Spark
[![](https://markdown-videos-api.jorgenkh.no/youtube/9qrDsY_2COo)](https://youtu.be/9qrDsY_2COo&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=59)
* :movie_camera: 5.4.3 Joins in Spark
[![](https://markdown-videos-api.jorgenkh.no/youtube/lu7TrqAWuH4)](https://youtu.be/lu7TrqAWuH4&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=60)
## 5.5 (Optional) Resilient Distributed Datasets
* :movie_camera: 5.5.1 Operations on Spark RDDs
[![](https://markdown-videos-api.jorgenkh.no/youtube/Bdu-xIrF3OM)](https://youtu.be/Bdu-xIrF3OM&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=61)
* :movie_camera: 5.5.2 Spark RDD mapPartition
[![](https://markdown-videos-api.jorgenkh.no/youtube/k3uB2K99roI)](https://youtu.be/k3uB2K99roI&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=62)
## 5.6 Running Spark in the Cloud
* :movie_camera: 5.6.1 Connecting to Google Cloud Storage
[![](https://markdown-videos-api.jorgenkh.no/youtube/Yyz293hBVcQ)](https://youtu.be/Yyz293hBVcQ&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=63)
* :movie_camera: 5.6.2 Creating a Local Spark Cluster
[![](https://markdown-videos-api.jorgenkh.no/youtube/HXBwSlXo5IA)](https://youtu.be/HXBwSlXo5IA&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=64)
* :movie_camera: 5.6.3 Setting up a Dataproc Cluster
[![](https://markdown-videos-api.jorgenkh.no/youtube/osAiAYahvh8)](https://youtu.be/osAiAYahvh8&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=65)
* :movie_camera: 5.6.4 Connecting Spark to Big Query
[![](https://markdown-videos-api.jorgenkh.no/youtube/HIm2BOj8C0Q)](https://youtu.be/HIm2BOj8C0Q&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=66)
# Homework
* [2024 Homework](../cohorts/2024/05-batch/homework.md)
# Community notes
Did you take notes? You can share them here.
* [Notes by Alvaro Navas](https://github.com/ziritrion/dataeng-zoomcamp/blob/main/notes/5_batch_processing.md)
* [Sandy's DE Learning Blog](https://learningdataengineering540969211.wordpress.com/2022/02/24/week-5-de-zoomcamp-5-2-1-installing-spark-on-linux/)
* [Notes by Alain Boisvert](https://github.com/boisalai/de-zoomcamp-2023/blob/main/week5.md)
* [Alternative : Using docker-compose to launch spark by rafik](https://gist.github.com/rafik-rahoui/f98df941c4ccced9c46e9ccbdef63a03)
* [Marcos Torregrosa's blog (spanish)](https://www.n4gash.com/2023/data-engineering-zoomcamp-semana-5-batch-spark)
* [Notes by Victor Padilha](https://github.com/padilha/de-zoomcamp/tree/master/week5)
* [Notes by Oscar Garcia](https://github.com/ozkary/Data-Engineering-Bootcamp/tree/main/Step5-Batch-Processing)
* [Notes by HongWei](https://github.com/hwchua0209/data-engineering-zoomcamp-submission/blob/main/05-batch-processing/README.md)
* [2024 videos transcript](https://drive.google.com/drive/folders/1XMmP4H5AMm1qCfMFxc_hqaPGw31KIVcb?usp=drive_link) by Maria Fisher
* Add your notes here (above this line)

129
06-streaming/README.md
View File

@ -1,129 +0,0 @@
# Week 6: Stream Processing
# Code structure
* [Java examples](java)
* [Python examples](python)
* [KSQLD examples](ksqldb)
## Confluent cloud setup
Confluent cloud provides a free 30 days trial for, you can signup [here](https://www.confluent.io/confluent-cloud/tryfree/)
## Introduction to Stream Processing
- [Slides](https://docs.google.com/presentation/d/1bCtdCba8v1HxJ_uMm9pwjRUC-NAMeB-6nOG2ng3KujA/edit?usp=sharing)
- :movie_camera: 6.0.1 Introduction
[![](https://markdown-videos-api.jorgenkh.no/youtube/hfvju3iOIP0)](https://youtu.be/hfvju3iOIP0&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=67)
- :movie_camera: 6.0.2 What is stream processing
[![](https://markdown-videos-api.jorgenkh.no/youtube/WxTxKGcfA-k)](https://youtu.be/WxTxKGcfA-k&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=68)
## Introduction to Kafka
- :movie_camera: 6.3 What is kafka?
[![](https://markdown-videos-api.jorgenkh.no/youtube/zPLZUDPi4AY)](https://youtu.be/zPLZUDPi4AY&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=69)
- :movie_camera: 6.4 Confluent cloud
[![](https://markdown-videos-api.jorgenkh.no/youtube/ZnEZFEYKppw)](https://youtu.be/ZnEZFEYKppw&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=70)
- :movie_camera: 6.5 Kafka producer consumer
[![](https://markdown-videos-api.jorgenkh.no/youtube/aegTuyxX7Yg)](https://youtu.be/aegTuyxX7Yg&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=71)
## Kafka Configuration
- :movie_camera: 6.6 Kafka configuration
[![](https://markdown-videos-api.jorgenkh.no/youtube/SXQtWyRpMKs)](https://youtu.be/SXQtWyRpMKs&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=72)
- [Kafka Configuration Reference](https://docs.confluent.io/platform/current/installation/configuration/)
## Kafka Streams
- [Slides](https://docs.google.com/presentation/d/1fVi9sFa7fL2ZW3ynS5MAZm0bRSZ4jO10fymPmrfTUjE/edit?usp=sharing)
- [Streams Concepts](https://docs.confluent.io/platform/current/streams/concepts.html)
- :movie_camera: 6.7 Kafka streams basics
[![](https://markdown-videos-api.jorgenkh.no/youtube/dUyA_63eRb0)](https://youtu.be/dUyA_63eRb0&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=73)
- :movie_camera: 6.8 Kafka stream join
[![](https://markdown-videos-api.jorgenkh.no/youtube/NcpKlujh34Y)](https://youtu.be/NcpKlujh34Y&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=74)
- :movie_camera: 6.9 Kafka stream testing
[![](https://markdown-videos-api.jorgenkh.no/youtube/TNx5rmLY8Pk)](https://youtu.be/TNx5rmLY8Pk&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=75)
- :movie_camera: 6.10 Kafka stream windowing
[![](https://markdown-videos-api.jorgenkh.no/youtube/r1OuLdwxbRc)](https://youtu.be/r1OuLdwxbRc&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=76)
- :movie_camera: 6.11 Kafka ksqldb & Connect
[![](https://markdown-videos-api.jorgenkh.no/youtube/DziQ4a4tn9Y)](https://youtu.be/DziQ4a4tn9Y&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=77)
- :movie_camera: 6.12 Kafka Schema registry
[![](https://markdown-videos-api.jorgenkh.no/youtube/tBY_hBuyzwI)](https://youtu.be/tBY_hBuyzwI&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=78)
## Faust - Python Stream Processing
- [Faust Documentation](https://faust.readthedocs.io/en/latest/index.html)
- [Faust vs Kafka Streams](https://faust.readthedocs.io/en/latest/playbooks/vskafka.html)
## Pyspark - Structured Streaming
Please follow the steps described under [pyspark-streaming](python/streams-example/pyspark/README.md)
- :movie_camera: 6.13 Kafka Streaming with Python
[![](https://markdown-videos-api.jorgenkh.no/youtube/BgAlVknDFlQ)](https://youtu.be/BgAlVknDFlQ&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=79)
- :movie_camera: 6.14 Pyspark Structured Streaming
[![](https://markdown-videos-api.jorgenkh.no/youtube/VIVr7KwRQmE)](https://youtu.be/VIVr7KwRQmE&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=80)
## Kafka Streams with JVM library
- [Confluent Kafka Streams](https://kafka.apache.org/documentation/streams/)
- [Scala Example](https://github.com/AnkushKhanna/kafka-helper/tree/master/src/main/scala/kafka/schematest)
## KSQL and ksqlDB
- [Introducing KSQL: Streaming SQL for Apache Kafka](https://www.confluent.io/blog/ksql-streaming-sql-for-apache-kafka/)
- [ksqlDB](https://ksqldb.io/)
## Kafka Connect
- [Making Sense of Stream Data](https://medium.com/analytics-vidhya/making-sense-of-stream-data-b74c1252a8f5)
## Docker
### Starting cluster
## Command line for Kafka
### Create topic
```bash
./bin/kafka-topics.sh --create --topic demo_1 --bootstrap-server localhost:9092 --partitions 2
```
## Homework
* [2024 Homework](../cohorts/2024/)
## Community notes
Did you take notes? You can share them here.
* [Notes by Alvaro Navas](https://github.com/ziritrion/dataeng-zoomcamp/blob/main/notes/6_streaming.md )
* [Marcos Torregrosa's blog (spanish)](https://www.n4gash.com/2023/data-engineering-zoomcamp-semana-6-stream-processing/)
* [Notes by Oscar Garcia](https://github.com/ozkary/Data-Engineering-Bootcamp/tree/main/Step6-Streaming)
* Add your notes here (above this line)

108
06-streaming/python/redpanda_example/README.md
View File

@ -1,108 +0,0 @@
# Basic PubSub example with Redpanda
The aim of this module is to have a good grasp on the foundation of these Kafka/Redpanda concepts, to be able to submit a capstone project using streaming:
- clusters
- brokers
- topics
- producers
- consumers and consumer groups
- data serialization and deserialization
- replication and retention
- offsets
- consumer-groups
-
## 1. Pre-requisites
If you have been following the [module-06](./../../../06-streaming/README.md) videos, you might already have installed the `kafka-python` library, so you can move on to [Docker](#2-docker) section.
If you have not, this is the only package you need to install in your virtual environment for this Redpanda lesson.
1. activate your environment
2. `pip install kafka-python`
## 2. Docker
Start a Redpanda cluster. Redpanda is a single binary image, so it is very easy to start learning kafka concepts with Redpanda.
```bash
cd 06-streaming/python/redpanda_example/
docker-compose up -d
```
## 3. Set RPK alias
Redpanda has a console command `rpk` which means `Redpanda keeper`, the CLI tool that ships with Redpanda and is already available in the Docker image.
Set the following `rpk` alias so we can use it from our terminal, without having to open a Docker interactive terminal. We can use this `rpk` alias directly in our terminal.
```bash
alias rpk="docker exec -ti redpanda-1 rpk"
rpk version
```
At this time, the verion is shown as `v23.2.26 (rev 328d83a06e)`. The important version munber is the major one `v23` following the versioning semantics `major.minor[.build[.revision]]`, to ensure that you get the same results as whatever is shared in this document.
> [!TIP]
> If you're reading this after Mar, 2024 and want to update the Docker file to use the latest Redpanda images, just visit [Docker hub](https://hub.docker.com/r/vectorized/redpanda/tags), and paste the new version number.
## 4. Kafka Producer - Consumer Examples
To run the producer-consumer examples, open 2 shell terminals in 2 side-by-side tabs and run following commands. Be sure to activate your virtual environment in each terminal.
```bash
# Start consumer script, in 1st terminal tab
python -m consumer.py
# Start producer script, in 2nd terminal tab
python -m producer.py
```
Run the `python -m producer.py` command again (and again) to observe that the `consumer` worker tab would automatically consume messages in real-time when new `events` occur
## 5. Redpanda UI
You can also see the clusters, topics, etc from the Redpanda Console UI via your browser at [http://localhost:8080](http://localhost:8080)
## 6. rpk commands glossary
Visit [get-started-rpk blog post](https://redpanda.com/blog/get-started-rpk-manage-streaming-data-clusters) for more.
```bash
# set alias for rpk
alias rpk="docker exec -ti redpanda-1 rpk"
# get info on cluster
rpk cluster info
# create topic_name with m partitions and n replication factor
rpk topic create [topic_name] --partitions m --replicas n
# get list of available topics, without extra details and with details
rpk topic list
rpk topic list --detailed
# inspect topic config
rpk topic describe [topic_name]
# consume [topic_name]
rpk topic consume [topic_name]
# list the consumer groups in a Redpanda cluster
rpk group list
# get additional information about a consumer group, from above listed result
rpk group describe my-group
```
## 7. Additional Resources
Redpanda Univerity (needs a Redpanda account and it is free to enrol and do the course(s))
- [RP101: Getting Started with Redpanda](https://university.redpanda.com/courses/hands-on-redpanda-getting-started)
- [RP102: Stream Processing with Redpanda](https://university.redpanda.com/courses/take/hands-on-redpanda-stream-processing/lessons/37830192-intro)
- [SF101: Streaming Fundamentals](https://university.redpanda.com/courses/streaming-fundamentals)
- [SF102: Kafka building blocks](https://university.redpanda.com/courses/kafka-building-blocks)
If you feel that you already have a good foundational basis on Streaming and Kafka, feel free to skip these supplementary courses.

48
06-streaming/python/redpanda_example/consumer.py
View File

@ -1,48 +0,0 @@
import os
from typing import Dict, List
from json import loads
from kafka import KafkaConsumer
from ride import Ride
from settings import BOOTSTRAP_SERVERS, KAFKA_TOPIC
class JsonConsumer:
def __init__(self, props: Dict):
self.consumer = KafkaConsumer(**props)
def consume_from_kafka(self, topics: List[str]):
self.consumer.subscribe(topics)
print('Consuming from Kafka started')
print('Available topics to consume: ', self.consumer.subscription())
while True:
try:
# SIGINT can't be handled when polling, limit timeout to 1 second.
message = self.consumer.poll(1.0)
if message is None or message == {}:
continue
for message_key, message_value in message.items():
for msg_val in message_value:
print(msg_val.key, msg_val.value)
except KeyboardInterrupt:
break
self.consumer.close()
if __name__ == '__main__':
config = {
'bootstrap_servers': BOOTSTRAP_SERVERS,
'auto_offset_reset': 'earliest',
'enable_auto_commit': True,
'key_deserializer': lambda key: int(key.decode('utf-8')),
'value_deserializer': lambda x: loads(x.decode('utf-8'), object_hook=lambda d: Ride.from_dict(d)),
'group_id': 'consumer.group.id.json-example.1',
}
json_consumer = JsonConsumer(props=config)
json_consumer.consume_from_kafka(topics=[KAFKA_TOPIC])
# There's no schema in JSON format, so if the schema changes and one column is removed or new one added or the data types is changed, the Ride class would still work and produce-consume messages would still run without a hitch.
# But the issue is in the downstream Analytics as the dataset would no longer have that column and the dashboards would thus fail. Therefore, the trust in our data and processes would erodes.

90
06-streaming/python/redpanda_example/docker-compose.yaml
View File

@ -1,90 +0,0 @@
version: '3.7'
services:
# Redpanda cluster
redpanda-1:
image: docker.redpanda.com/redpandadata/redpanda:v23.2.26
container_name: redpanda-1
command:
- redpanda
- start
- --smp
- '1'
- --reserve-memory
- 0M
- --overprovisioned
- --node-id
- '1'
- --kafka-addr
- PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
- --advertise-kafka-addr
- PLAINTEXT://redpanda-1:29092,OUTSIDE://localhost:9092
- --pandaproxy-addr
- PLAINTEXT://0.0.0.0:28082,OUTSIDE://0.0.0.0:8082
- --advertise-pandaproxy-addr
- PLAINTEXT://redpanda-1:28082,OUTSIDE://localhost:8082
- --rpc-addr
- 0.0.0.0:33145
- --advertise-rpc-addr
- redpanda-1:33145
ports:
# - 8081:8081
- 8082:8082
- 9092:9092
- 9644:9644
- 28082:28082
- 29092:29092
# Want a two node Redpanda cluster? Uncomment this block :)
# redpanda-2:
# image: docker.redpanda.com/redpandadata/redpanda:v23.1.1
# container_name: redpanda-2
# command:
# - redpanda
# - start
# - --smp
# - '1'
# - --reserve-memory
# - 0M
# - --overprovisioned
# - --node-id
# - '2'
# - --seeds
# - redpanda-1:33145
# - --kafka-addr
# - PLAINTEXT://0.0.0.0:29093,OUTSIDE://0.0.0.0:9093
# - --advertise-kafka-addr
# - PLAINTEXT://redpanda-2:29093,OUTSIDE://localhost:9093
# - --pandaproxy-addr
# - PLAINTEXT://0.0.0.0:28083,OUTSIDE://0.0.0.0:8083
# - --advertise-pandaproxy-addr
# - PLAINTEXT://redpanda-2:28083,OUTSIDE://localhost:8083
# - --rpc-addr
# - 0.0.0.0:33146
# - --advertise-rpc-addr
# - redpanda-2:33146
# ports:
# - 8083:8083
# - 9093:9093
redpanda-console:
image: docker.redpanda.com/redpandadata/console:v2.2.2
container_name: redpanda-console
entrypoint: /bin/sh
command: -c "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console"
environment:
CONFIG_FILEPATH: /tmp/config.yml
CONSOLE_CONFIG_FILE: |
kafka:
brokers: ["redpanda-1:29092"]
schemaRegistry:
enabled: false
redpanda:
adminApi:
enabled: true
urls: ["http://redpanda-1:9644"]
connect:
enabled: false
ports:
- 8080:8080
depends_on:
- redpanda-1

44
06-streaming/python/redpanda_example/producer.py
View File

@ -1,44 +0,0 @@
import csv
import json
from typing import List, Dict
from kafka import KafkaProducer
from kafka.errors import KafkaTimeoutError
from ride import Ride
from settings import BOOTSTRAP_SERVERS, INPUT_DATA_PATH, KAFKA_TOPIC
class JsonProducer(KafkaProducer):
def __init__(self, props: Dict):
self.producer = KafkaProducer(**props)
@staticmethod
def read_records(resource_path: str):
records = []
with open(resource_path, 'r') as f:
reader = csv.reader(f)
header = next(reader) # skip the header row
for row in reader:
records.append(Ride(arr=row))
return records
def publish_rides(self, topic: str, messages: List[Ride]):
for ride in messages:
try:
record = self.producer.send(topic=topic, key=ride.pu_location_id, value=ride)
print('Record {} successfully produced at offset {}'.format(ride.pu_location_id, record.get().offset))
except KafkaTimeoutError as e:
print(e.__str__())
if __name__ == '__main__':
# Config Should match with the KafkaProducer expectation
# kafka expects binary format for the key-value pair
config = {
'bootstrap_servers': BOOTSTRAP_SERVERS,
'key_serializer': lambda key: str(key).encode(),
'value_serializer': lambda x: json.dumps(x.__dict__, default=str).encode('utf-8')
}
producer = JsonProducer(props=config)
rides = producer.read_records(resource_path=INPUT_DATA_PATH)
producer.publish_rides(topic=KAFKA_TOPIC, messages=rides)

52
06-streaming/python/redpanda_example/ride.py
View File

@ -1,52 +0,0 @@
from typing import List, Dict
from decimal import Decimal
from datetime import datetime
class Ride:
def __init__(self, arr: List[str]):
self.vendor_id = arr[0]
self.tpep_pickup_datetime = datetime.strptime(arr[1], "%Y-%m-%d %H:%M:%S"),
self.tpep_dropoff_datetime = datetime.strptime(arr[2], "%Y-%m-%d %H:%M:%S"),
self.passenger_count = int(arr[3])
self.trip_distance = Decimal(arr[4])
self.rate_code_id = int(arr[5])
self.store_and_fwd_flag = arr[6]
self.pu_location_id = int(arr[7])
self.do_location_id = int(arr[8])
self.payment_type = arr[9]
self.fare_amount = Decimal(arr[10])
self.extra = Decimal(arr[11])
self.mta_tax = Decimal(arr[12])
self.tip_amount = Decimal(arr[13])
self.tolls_amount = Decimal(arr[14])
self.improvement_surcharge = Decimal(arr[15])
self.total_amount = Decimal(arr[16])
self.congestion_surcharge = Decimal(arr[17])
@classmethod
def from_dict(cls, d: Dict):
return cls(arr=[
d['vendor_id'],
d['tpep_pickup_datetime'][0],
d['tpep_dropoff_datetime'][0],
d['passenger_count'],
d['trip_distance'],
d['rate_code_id'],
d['store_and_fwd_flag'],
d['pu_location_id'],
d['do_location_id'],
d['payment_type'],
d['fare_amount'],
d['extra'],
d['mta_tax'],
d['tip_amount'],
d['tolls_amount'],
d['improvement_surcharge'],
d['total_amount'],
d['congestion_surcharge'],
]
)
def __repr__(self):
return f'{self.__class__.__name__}: {self.__dict__}'

4
06-streaming/python/redpanda_example/settings.py
View File

@ -1,4 +0,0 @@
INPUT_DATA_PATH = '../resources/rides.csv'
BOOTSTRAP_SERVERS = ['localhost:9092']
KAFKA_TOPIC = 'rides_json'

46
06-streaming/python/streams-example/redpanda/README.md
View File

@ -1,46 +0,0 @@
# Running PySpark Streaming with Redpanda
### 1. Prerequisite
It is important to create network and volume as described in the document. Therefore please ensure, your volume and network are created correctly.
```bash
docker volume ls # should list hadoop-distributed-file-system
docker network ls # should list kafka-spark-network
```
### 2. Create Docker Network & Volume
If you have not followed any other examples, and above `ls` steps shows no output, create them now.
```bash
# Create Network
docker network create kafka-spark-network
# Create Volume
docker volume create --name=hadoop-distributed-file-system
```
### Running Producer and Consumer
```bash
# Run producer
python producer.py
# Run consumer with default settings
python consumer.py
# Run consumer for specific topic
python consumer.py --topic <topic-name>
```
### Running Streaming Script
spark-submit script ensures installation of necessary jars before running the streaming.py
```bash
./spark-submit.sh streaming.py
```
### Additional Resources
- [Structured Streaming Programming Guide](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#structured-streaming-programming-guide)
- [Structured Streaming + Kafka Integration](https://spark.apache.org/docs/latest/structured-streaming-kafka-integration.html#structured-streaming-kafka-integration-guide-kafka-broker-versio)

47
06-streaming/python/streams-example/redpanda/consumer.py
View File

@ -1,47 +0,0 @@
import argparse
from typing import Dict, List
from kafka import KafkaConsumer
from settings import BOOTSTRAP_SERVERS, CONSUME_TOPIC_RIDES_CSV
class RideCSVConsumer:
def __init__(self, props: Dict):
self.consumer = KafkaConsumer(**props)
def consume_from_kafka(self, topics: List[str]):
self.consumer.subscribe(topics=topics)
print('Consuming from Kafka started')
print('Available topics to consume: ', self.consumer.subscription())
while True:
try:
# SIGINT can't be handled when polling, limit timeout to 1 second.
msg = self.consumer.poll(1.0)
if msg is None or msg == {}:
continue
for msg_key, msg_values in msg.items():
for msg_val in msg_values:
print(f'Key:{msg_val.key}-type({type(msg_val.key)}), '
f'Value:{msg_val.value}-type({type(msg_val.value)})')
except KeyboardInterrupt:
break
self.consumer.close()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Kafka Consumer')
parser.add_argument('--topic', type=str, default=CONSUME_TOPIC_RIDES_CSV)
args = parser.parse_args()
topic = args.topic
config = {
'bootstrap_servers': [BOOTSTRAP_SERVERS],
'auto_offset_reset': 'earliest',
'enable_auto_commit': True,
'key_deserializer': lambda key: int(key.decode('utf-8')),
'value_deserializer': lambda value: value.decode('utf-8'),
'group_id': 'consumer.group.id.csv-example.1',
}
csv_consumer = RideCSVConsumer(props=config)
csv_consumer.consume_from_kafka(topics=[topic])

104
06-streaming/python/streams-example/redpanda/docker-compose.yaml
View File

@ -1,104 +0,0 @@
version: '3.7'
volumes:
shared-workspace:
name: "hadoop-distributed-file-system"
driver: local
networks:
default:
name: kafka-spark-network
external: true
services:
# Redpanda cluster
redpanda-1:
image: docker.redpanda.com/redpandadata/redpanda:v23.2.26
container_name: redpanda-1
command:
- redpanda
- start
- --smp
- '1'
- --reserve-memory
- 0M
- --overprovisioned
- --node-id
- '1'
- --kafka-addr
- PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
- --advertise-kafka-addr
- PLAINTEXT://redpanda-1:29092,OUTSIDE://localhost:9092
- --pandaproxy-addr
- PLAINTEXT://0.0.0.0:28082,OUTSIDE://0.0.0.0:8082
- --advertise-pandaproxy-addr
- PLAINTEXT://redpanda-1:28082,OUTSIDE://localhost:8082
- --rpc-addr
- 0.0.0.0:33145
- --advertise-rpc-addr
- redpanda-1:33145
ports:
# - 8081:8081
- 8082:8082
- 9092:9092
- 9644:9644
- 28082:28082
- 29092:29092
volumes:
- shared-workspace:/opt/workspace
# Want a two node Redpanda cluster? Uncomment this block :)
redpanda-2:
image: docker.redpanda.com/redpandadata/redpanda:v23.1.1
container_name: redpanda-2
command:
- redpanda
- start
- --smp
- '1'
- --reserve-memory
- 0M
- --overprovisioned
- --node-id
- '2'
- --seeds
- redpanda-1:33145
- --kafka-addr
- PLAINTEXT://0.0.0.0:29093,OUTSIDE://0.0.0.0:9093
- --advertise-kafka-addr
- PLAINTEXT://redpanda-2:29093,OUTSIDE://localhost:9093
- --pandaproxy-addr
- PLAINTEXT://0.0.0.0:28083,OUTSIDE://0.0.0.0:8083
- --advertise-pandaproxy-addr
- PLAINTEXT://redpanda-2:28083,OUTSIDE://localhost:8083
- --rpc-addr
- 0.0.0.0:33146
- --advertise-rpc-addr
- redpanda-2:33146
ports:
- 8083:8083
- 9093:9093
volumes:
- shared-workspace:/opt/workspace
redpanda-console:
image: docker.redpanda.com/redpandadata/console:v2.2.2
container_name: redpanda-console
entrypoint: /bin/sh
command: -c "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console"
environment:
CONFIG_FILEPATH: /tmp/config.yml
CONSOLE_CONFIG_FILE: |
kafka:
brokers: ["redpanda-1:29092"]
schemaRegistry:
enabled: false
redpanda:
adminApi:
enabled: true
urls: ["http://redpanda-1:9644"]
connect:
enabled: false
ports:
- 8080:8080
depends_on:
- redpanda-1
volumes:
- shared-workspace:/opt/workspace

62
06-streaming/python/streams-example/redpanda/producer.py
View File

@ -1,62 +0,0 @@
import csv
from time import sleep
from typing import Dict
from kafka import KafkaProducer
from settings import BOOTSTRAP_SERVERS, INPUT_DATA_PATH, PRODUCE_TOPIC_RIDES_CSV
def delivery_report(err, msg):
if err is not None:
print("Delivery failed for record {}: {}".format(msg.key(), err))
return
print('Record {} successfully produced to {} [{}] at offset {}'.format(
msg.key(), msg.topic(), msg.partition(), msg.offset()))
class RideCSVProducer:
def __init__(self, props: Dict):
self.producer = KafkaProducer(**props)
# self.producer = Producer(producer_props)
@staticmethod
def read_records(resource_path: str):
records, ride_keys = [], []
i = 0
with open(resource_path, 'r') as f:
reader = csv.reader(f)
header = next(reader) # skip the header
for row in reader:
# vendor_id, passenger_count, trip_distance, payment_type, total_amount
records.append(f'{row[0]}, {row[1]}, {row[2]}, {row[3]}, {row[4]}, {row[9]}, {row[16]}')
ride_keys.append(str(row[0]))
i += 1
if i == 5:
break
return zip(ride_keys, records)
def publish(self, topic: str, records: [str, str]):
for key_value in records:
key, value = key_value
try:
self.producer.send(topic=topic, key=key, value=value)
print(f"Producing record for <key: {key}, value:{value}>")
except KeyboardInterrupt:
break
except Exception as e:
print(f"Exception while producing record - {value}: {e}")
self.producer.flush()
sleep(1)
if __name__ == "__main__":
config = {
'bootstrap_servers': [BOOTSTRAP_SERVERS],
'key_serializer': lambda x: x.encode('utf-8'),
'value_serializer': lambda x: x.encode('utf-8')
}
producer = RideCSVProducer(props=config)
ride_records = producer.read_records(resource_path=INPUT_DATA_PATH)
print(ride_records)
producer.publish(topic=PRODUCE_TOPIC_RIDES_CSV, records=ride_records)

18
06-streaming/python/streams-example/redpanda/settings.py
View File

@ -1,18 +0,0 @@
import pyspark.sql.types as T
INPUT_DATA_PATH = '../../resources/rides.csv'
BOOTSTRAP_SERVERS = 'localhost:9092'
TOPIC_WINDOWED_VENDOR_ID_COUNT = 'vendor_counts_windowed'
PRODUCE_TOPIC_RIDES_CSV = CONSUME_TOPIC_RIDES_CSV = 'rides_csv'
RIDE_SCHEMA = T.StructType(
[T.StructField("vendor_id", T.IntegerType()),
T.StructField('tpep_pickup_datetime', T.TimestampType()),
T.StructField('tpep_dropoff_datetime', T.TimestampType()),
T.StructField("passenger_count", T.IntegerType()),
T.StructField("trip_distance", T.FloatType()),
T.StructField("payment_type", T.IntegerType()),
T.StructField("total_amount", T.FloatType()),
])

20
06-streaming/python/streams-example/redpanda/spark-submit.sh
View File

@ -1,20 +0,0 @@
# Submit Python code to SparkMaster
if [ $# -lt 1 ]
then
echo "Usage: $0 <pyspark-job.py> [ executor-memory ]"
echo "(specify memory in string format such as \"512M\" or \"2G\")"
exit 1
fi
PYTHON_JOB=$1
if [ -z $2 ]
then
EXEC_MEM="1G"
else
EXEC_MEM=$2
fi
spark-submit --master spark://localhost:7077 --num-executors 2 \
--executor-memory $EXEC_MEM --executor-cores 1 \
--packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.1,org.apache.spark:spark-avro_2.12:3.5.1,org.apache.spark:spark-streaming-kafka-0-10_2.12:3.5.1 \
$PYTHON_JOB

1105
06-streaming/python/streams-example/redpanda/streaming-notebook.ipynb
View File

File diff suppressed because it is too large Load Diff

127
06-streaming/python/streams-example/redpanda/streaming.py
View File

@ -1,127 +0,0 @@
from pyspark.sql import SparkSession
import pyspark.sql.functions as F
from settings import RIDE_SCHEMA, CONSUME_TOPIC_RIDES_CSV, TOPIC_WINDOWED_VENDOR_ID_COUNT
def read_from_kafka(consume_topic: str):
# Spark Streaming DataFrame, connect to Kafka topic served at host in bootrap.servers option
df_stream = spark \
.readStream \
.format("kafka") \
.option("kafka.bootstrap.servers", "localhost:9092,broker:29092") \
.option("subscribe", consume_topic) \
.option("startingOffsets", "earliest") \
.option("checkpointLocation", "checkpoint") \
.load()
return df_stream
def parse_ride_from_kafka_message(df, schema):
""" take a Spark Streaming df and parse value col based on <schema>, return streaming df cols in schema """
assert df.isStreaming is True, "DataFrame doesn't receive streaming data"
df = df.selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
# split attributes to nested array in one Column
col = F.split(df['value'], ', ')
# expand col to multiple top-level columns
for idx, field in enumerate(schema):
df = df.withColumn(field.name, col.getItem(idx).cast(field.dataType))
return df.select([field.name for field in schema])
def sink_console(df, output_mode: str = 'complete', processing_time: str = '5 seconds'):
write_query = df.writeStream \
.outputMode(output_mode) \
.trigger(processingTime=processing_time) \
.format("console") \
.option("truncate", False) \
.start()
return write_query # pyspark.sql.streaming.StreamingQuery
def sink_memory(df, query_name, query_template):
query_df = df \
.writeStream \
.queryName(query_name) \
.format("memory") \
.start()
query_str = query_template.format(table_name=query_name)
query_results = spark.sql(query_str)
return query_results, query_df
def sink_kafka(df, topic):
write_query = df.writeStream \
.format("kafka") \
.option("kafka.bootstrap.servers", "localhost:9092,broker:29092") \
.outputMode('complete') \
.option("topic", topic) \
.option("checkpointLocation", "checkpoint") \
.start()
return write_query
def prepare_df_to_kafka_sink(df, value_columns, key_column=None):
columns = df.columns
df = df.withColumn("value", F.concat_ws(', ', *value_columns))
if key_column:
df = df.withColumnRenamed(key_column, "key")
df = df.withColumn("key", df.key.cast('string'))
return df.select(['key', 'value'])
def op_groupby(df, column_names):
df_aggregation = df.groupBy(column_names).count()
return df_aggregation
def op_windowed_groupby(df, window_duration, slide_duration):
df_windowed_aggregation = df.groupBy(
F.window(timeColumn=df.tpep_pickup_datetime, windowDuration=window_duration, slideDuration=slide_duration),
df.vendor_id
).count()
return df_windowed_aggregation
if __name__ == "__main__":
spark = SparkSession.builder.appName('streaming-examples').getOrCreate()
spark.sparkContext.setLogLevel('WARN')
# read_streaming data
df_consume_stream = read_from_kafka(consume_topic=CONSUME_TOPIC_RIDES_CSV)
print(df_consume_stream.printSchema())
# parse streaming data
df_rides = parse_ride_from_kafka_message(
df_consume_stream,
RIDE_SCHEMA
)
print(df_rides.printSchema())
sink_console(df_rides, output_mode='append')
df_trip_count_by_vendor_id = op_groupby(df_rides, ['vendor_id'])
df_trip_count_by_pickup_date_vendor_id = op_windowed_groupby(
df_rides,
window_duration="10 minutes",
slide_duration='5 minutes'
)
# write the output out to the console for debugging / testing
sink_console(df_trip_count_by_vendor_id)
# write the output to the kafka topic
df_trip_count_messages = prepare_df_to_kafka_sink(
df=df_trip_count_by_pickup_date_vendor_id,
value_columns=['count'],
key_column='vendor_id'
)
kafka_sink_query = sink_kafka(
df=df_trip_count_messages,
topic=TOPIC_WINDOWED_VENDOR_ID_COUNT
)
spark.streams.awaitAnyTermination()

127
README.md
View File

@ -13,15 +13,13 @@
Syllabus
* [Module 1: Containerization and Infrastructure as Code](#module-1-containerization-and-infrastructure-as-code)
* [Module 1: Introduction & Prerequisites](#module-1-introduction--prerequisites)
* [Module 2: Workflow Orchestration](#module-2-workflow-orchestration)
* [Workshop 1: Data Ingestion](#workshop-1-data-ingestion)
* [Module 3: Data Warehouse](#module-3-data-warehouse)
* [Module 4: Analytics Engineering](#module-4-analytics-engineering)
* [Module 5: Batch processing](#module-5-batch-processing)
* [Module 6: Streaming](#module-6-streaming)
* [Workshop 2: Stream Processing with SQL](#workshop-2-stream-processing-with-sql)
* [Project](#project)
* [Module 7: Project](#module-7-project)
## Taking the course
@ -29,8 +27,7 @@ Syllabus
* **Start**: 15 January 2024 (Monday) at 17:00 CET
* **Registration link**: https://airtable.com/shr6oVXeQvSI5HuWD
* [Cohort folder](cohorts/2024/) with homeworks and deadlines
* [Launch stream with course overview](https://www.youtube.com/live/AtRhA-NfS24?si=5JzA_E8BmJjiLi8l)
* [Cohort folder](cohorts/2024/) with homeworks and deadlines
### Self-paced mode
@ -46,11 +43,10 @@ can take the course at your own pace
## Syllabus
> **Note:** NYC TLC changed the format of the data we use to parquet.
> In the course we still use the CSV files accessible [here](https://github.com/DataTalksClub/nyc-tlc-data).
> **Note:** NYC TLC changed the format of the data we use to parquet. But you can still access
> the csv files [here](https://github.com/DataTalksClub/nyc-tlc-data).
### [Module 1: Containerization and Infrastructure as Code](01-docker-terraform/)
### [Module 1: Introduction & Prerequisites](week_1_basics_n_setup)
* Course overview
* Introduction to GCP
@ -60,44 +56,34 @@ can take the course at your own pace
* Preparing the environment for the course
* Homework
[More details](01-docker-terraform/)
[More details](week_1_basics_n_setup)
### [Module 2: Workflow Orchestration](02-workflow-orchestration/)
### [Module 2: Workflow Orchestration](week_2_workflow_orchestration/)
* Data Lake
* Workflow orchestration
* Workflow orchestration with Mage
* Homework
[More details](02-workflow-orchestration/)
[More details](week_2_workflow_orchestration/)
### [Workshop 1: Data Ingestion](cohorts/2024/workshops/dlt.md)
### [Module 3: Data Warehouse](week_3_data_warehouse)
* Reading from apis
* Building scalable pipelines
* Normalising data
* Incremental loading
* Homework
[More details](cohorts/2024/workshops/dlt.md)
### [Module 3: Data Warehouse](03-data-warehouse/)
* Data Warehouse
* BigQuery
* Partitioning and clustering
* BigQuery best practices
* Internals of BigQuery
* Integrating BigQuery with Airflow
* BigQuery Machine Learning
[More details](03-data-warehouse/)
[More details](week_3_data_warehouse)
### [Module 4: Analytics engineering](04-analytics-engineering/)
### [Module 4: Analytics engineering](week_4_analytics_engineering/)
* Basics of analytics engineering
* dbt (data build tool)
@ -109,10 +95,10 @@ can take the course at your own pace
* Visualizing the data with google data studio and metabase
[More details](04-analytics-engineering/)
[More details](week_4_analytics_engineering)
### [Module 5: Batch processing](05-batch/)
### [Module 5: Batch processing](week_5_batch_processing)
* Batch processing
* What is Spark
@ -120,36 +106,61 @@ can take the course at your own pace
* Spark SQL
* Internals: GroupBy and joins
[More details](05-batch/)
[More details](week_5_batch_processing)
### [Module 6: Streaming](06-streaming/)
### [Module 6: Streaming](week_6_stream_processing)
* Introduction to Kafka
* Schemas (avro)
* Kafka Streams
* Kafka Connect and KSQL
[More details](06-streaming/)
[More details](week_6_stream_processing)
### [Workshop 2: Stream Processing with SQL](cohorts/2024/workshops/rising-wave.md)
[More details](cohorts/2024/workshops/rising-wave.md)
### [Project](projects)
### [Module 7: Project](week_7_project)
Putting everything we learned to practice
* Week 1 and 2: working on your project
* Week 3: reviewing your peers
* Week 7 and 8: working on your project
* Week 9: reviewing your peers
[More details](projects)
[More details](week_7_project)
### Course UI
Alternatively, you can access this course using the provided UI app, the app provides a user-friendly interface for navigating through the course material.
* Visit the following link: [DE Zoomcamp UI](https://dezoomcamp.streamlit.app/)
![dezoomcamp-ui](https://github.com/DataTalksClub/data-engineering-zoomcamp/assets/66017329/4466d2bc-3728-4fca-8e9e-b1c6be30a430)
### Asking for help in Slack
The best way to get support is to use [DataTalks.Club's Slack](https://datatalks.club/slack.html). Join the [`#course-data-engineering`](https://app.slack.com/client/T01ATQK62F8/C01FABYF2RG) channel.
To make discussions in Slack more organized:
* Follow [these recommendations](asking-questions.md) when asking for help
* Read the [DataTalks.Club community guidelines](https://datatalks.club/slack/guidelines.html)
## Overview
<img src="images/architecture/arch_v3_workshops.jpg" />
### Architecture diagram
<img src="images/architecture/arch_2.png"/>
### Technologies
* *Google Cloud Platform (GCP)*: Cloud-based auto-scaling platform by Google
* *Google Cloud Storage (GCS)*: Data Lake
* *BigQuery*: Data Warehouse
* *Terraform*: Infrastructure-as-Code (IaC)
* *Docker*: Containerization
* *SQL*: Data Analysis & Exploration
* *Mage*: Workflow Orchestration
* *dbt*: Data Transformation
* *Spark*: Distributed Processing
* *Kafka*: Streaming
### Prerequisites
@ -166,24 +177,18 @@ Prior experience with data engineering is not required.
- [Ankush Khanna](https://linkedin.com/in/ankushkhanna2)
- [Victoria Perez Mola](https://www.linkedin.com/in/victoriaperezmola/)
- [Alexey Grigorev](https://linkedin.com/in/agrigorev)
- [Matt Palmer](https://www.linkedin.com/in/matt-palmer/)
- [Luis Oliveira](https://www.linkedin.com/in/lgsoliveira/)
- [Michael Shoemaker](https://www.linkedin.com/in/michaelshoemaker1/)
Past instructors:
## Tools
- [Sejal Vaidya](https://www.linkedin.com/in/vaidyasejal/)
- [Irem Erturk](https://www.linkedin.com/in/iremerturk/)
For this course, you'll need to have the following software installed on your computer:
* Docker and Docker-Compose
* Python 3 (e.g. via [Anaconda](https://www.anaconda.com/products/individual))
* Google Cloud SDK
* Terraform
## Asking for help in Slack
See [Module 1](week_1_basics_n_setup) for more details about installing these tools
The best way to get support is to use [DataTalks.Club's Slack](https://datatalks.club/slack.html). Join the [`#course-data-engineering`](https://app.slack.com/client/T01ATQK62F8/C01FABYF2RG) channel.
To make discussions in Slack more organized:
* Follow [these recommendations](asking-questions.md) when asking for help
* Read the [DataTalks.Club community guidelines](https://datatalks.club/slack/guidelines.html)
@ -193,7 +198,7 @@ Thanks to the course sponsors for making it possible to run this course
<p align="center">
<a href="https://mage.ai/">
<img height="120" src="images/mage.svg">
<img height="150" src="images/mage.svg">
</a>
</p>
@ -204,14 +209,6 @@ Thanks to the course sponsors for making it possible to run this course
</a>
</p>
<p align="center">
<a href="https://risingwave.com/">
<img height="90" src="images/rising-wave.png">
</a>
</p>
Do you want to support our course and our community? Please reach out to [alexey@datatalks.club](alexey@datatalks.club)
## Star History
[![Star History Chart](https://api.star-history.com/svg?repos=DataTalksClub/data-engineering-zoomcamp&type=Date)](https://star-history.com/#DataTalksClub/data-engineering-zoomcamp&Date)

2
arch_diagram.md Normal file
View File

@ -0,0 +1,2 @@
![](images/architecture/arch_1.jpg)

37
asking-questions.md
View File

@ -7,14 +7,11 @@ To keep our discussion in Slack more organized, we ask you to follow these sugge
* First, review How to troubleshoot issues listed below.
* Before asking a question, check the [FAQ](https://docs.google.com/document/d/19bnYs80DwuUimHM65UV3sylsCn2j1vziPOwzBwQrebw/edit).
* Before asking a question review the [Slack Guidelines](#Ask-in-Slack).
* Before asking a question review the [Slack Guidlines](#Ask-in-Slack).
* If somebody helped you with your problem and it's not in [FAQ](https://docs.google.com/document/d/19bnYs80DwuUimHM65UV3sylsCn2j1vziPOwzBwQrebw/edit), please add it there.
It'll help other students.
* Zed Shaw (of the Learn the Hard Way series) has [a great post on how to help others help you](https://learncodethehardway.com/blog/03-how-to-ask-for-help/)
* Check [Stackoverflow guide on asking](https://stackoverflow.com/help/how-to-ask)
### How to troubleshoot issues
The first step is to try to solve the issue on you own; get use to solving problems. This will be a real life skill you need when employeed.
1. What does the error say? There will often be a description of the error or instructions on what is needed, I have even seen a link to the solution. Does it reference a specific line of your code?
@ -24,21 +21,21 @@ The first step is to try to solve the issue on you own; get use to solving probl
4. Check the tech’s documentation. Use its search if available or use the browsers search function.
5. Try uninstall (this may remove the bad actor) and reinstall of application or reimplementation of action. Don’t forget to restart the server/pc for reinstalls.
* Sometimes reinstalling fails to resolve the issue but works if you uninstall first.
6. Ask in Slack
7. Take a break and come back to it later. You will be amazed at how often you figure out the solution after letting your brain rest. Get some fresh air, workout, play a video game, watch a tv show, whatever allows your brain to not think about it for a little while or even until the next day.
8. Remember technology issues in real life sometimes take days or even weeks to resolve
### Asking in Slack
* Before asking a question, check the [FAQ](https://docs.google.com/document/d/19bnYs80DwuUimHM65UV3sylsCn2j1vziPOwzBwQrebw/edit).
* DO NOT use screenshots, especially don’t take pictures from a phone.
* DO NOT tag instructors, it may discourage others from helping you.
* Copy and past errors; if it’s long, just post it in a reply to your thread.
* Use ``` for formatting your code.
* Use the same thread for the conversation (that means reply to your own thread).
* DO NOT create multiple posts to discus the issue.
* You may create a new post if the issue reemerges down the road. Be sure to describe what has changed in the environment.
* Provide addition information in the same thread of the steps you have taken for resolution.
6. Post your question to Stackoverflow. Be sure to read the Stackoverflow guide on posting good questions.
* [Stackoverflow How To Ask Guide](https://stackoverflow.com/help/how-to-ask).
* This will be your real life ask an expert in the future (in addition to coworkers).
7. ##### Ask in Slack
* Before asking a question, check the [FAQ](https://docs.google.com/document/d/19bnYs80DwuUimHM65UV3sylsCn2j1vziPOwzBwQrebw/edit).
* DO NOT use screenshots, especially don’t take pictures from a phone.
* DO NOT tag instructors, it may discourage others from helping you.
* Copy and past errors; if it’s long, just post it in a reply to your thread.
* Use ``` for formatting your code.
* Use the same thread for the conversation (that means reply to your own thread).
* DO NOT create multiple posts to discus the issue.
* You may create a new post if the issue reemerges down the road. Be sure to describe what has changed in the environment.
* Provide addition information in the same thread of the steps you have taken for resolution.
8. Take a break and come back to it later. You will be amazed at how often you figure out the solution after letting your brain rest. Get some fresh air, workout, play a video game, watch a tv show, whatever allows your brain to not think about it for a little while or even until the next day.
9. Remember technology issues in real life sometimes take days or even weeks to resolve.

1
cohorts/2022/week_2_data_ingestion/README.md
View File

@ -68,5 +68,4 @@ Did you take notes? You can share them here.
* [Blog post by Isaac Kargar](https://kargarisaac.github.io/blog/data%20engineering/jupyter/2022/01/25/data-engineering-w2.html)
* [Blog, notes, walkthroughs by Sandy Behrens](https://learningdataengineering540969211.wordpress.com/2022/01/30/week-2-de-zoomcamp-2-3-2-ingesting-data-to-gcp-with-airflow/)
* [Notes from Apurva Hegde](https://github.com/apuhegde/Airflow-LocalExecutor-In-Docker#readme)
* [Notes from Vincenzo Galante](https://binchentso.notion.site/Data-Talks-Club-Data-Engineering-Zoomcamp-8699af8e7ff94ec49e6f9bdec8eb69fd)
* Add your notes here (above this line)

12
cohorts/2023/leaderboard.md
View File

@ -634,17 +634,5 @@ Links:
<td><a href="https://github.com/ChungWasawat/dtc_de_project">Project</a></td>
<td> <a href="https://www.linkedin.com/in/wasawat-boonyarittikit-b1698b179/"><img src="https://user-images.githubusercontent.com/875246/192300614-2ce22ed5-bbc4-4684-8098-d8128d71aac5.png" height="16em" /></a> <a href="https://github.com/ChungWasawat"><img src="https://user-images.githubusercontent.com/875246/192300611-a606521b-cb76-4090-be8e-7cc21752b996.png" height="16em" /></a></td>
<td></td>
</tr>
<tr>
<td>Fedor Faizov</td>
<td><a href="https://github.com/Fedrpi/de-zoomcamp-bandcamp-project">Project</a></td>
<td> <a href="https://www.linkedin.com/in/fedor-faizov-a75b32245/"><img src="https://user-images.githubusercontent.com/875246/192300614-2ce22ed5-bbc4-4684-8098-d8128d71aac5.png" height="16em" /></a> <a href="https://github.com/Fedrpi"><img src="https://user-images.githubusercontent.com/875246/192300611-a606521b-cb76-4090-be8e-7cc21752b996.png" height="16em" /></a></td>
<td><details>
<summary>More info</summary>
> Absolutly amazing course <3 </details></td>
</tr>
</table>

103
cohorts/2023/week_2_workflow_orchestration/README.md
View File

@ -1,103 +0,0 @@
## Week 2: Workflow Orchestration
Python code from videos is linked [below](#code-repository).
Also, if you find the commands too small to view in Kalise's videos, here's the [transcript with code for the second Prefect video](https://github.com/discdiver/prefect-zoomcamp/tree/main/flows/01_start) and the [fifth Prefect video](https://github.com/discdiver/prefect-zoomcamp/tree/main/flows/03_deployments).
### Data Lake (GCS)
* What is a Data Lake
* ELT vs. ETL
* Alternatives to components (S3/HDFS, Redshift, Snowflake etc.)
* [Video](https://www.youtube.com/watch?v=W3Zm6rjOq70&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* [Slides](https://docs.google.com/presentation/d/1RkH-YhBz2apIjYZAxUz2Uks4Pt51-fVWVN9CcH9ckyY/edit?usp=sharing)
### 1. Introduction to Workflow orchestration
* What is orchestration?
* Workflow orchestrators vs. other types of orchestrators
* Core features of a workflow orchestration tool
* Different types of workflow orchestration tools that currently exist
:movie_camera: [Video](https://www.youtube.com/watch?v=8oLs6pzHp68&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
### 2. Introduction to Prefect concepts
* What is Prefect?
* Installing Prefect
* Prefect flow
* Creating an ETL
* Prefect task
* Blocks and collections
* Orion UI
:movie_camera: [Video](https://www.youtube.com/watch?v=cdtN6dhp708&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
### 3. ETL with GCP & Prefect
* Flow 1: Putting data to Google Cloud Storage
:movie_camera: [Video](https://www.youtube.com/watch?v=W-rMz_2GwqQ&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
### 4. From Google Cloud Storage to Big Query
* Flow 2: From GCS to BigQuery
:movie_camera: [Video](https://www.youtube.com/watch?v=Cx5jt-V5sgE&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
### 5. Parametrizing Flow & Deployments
* Parametrizing the script from your flow
* Parameter validation with Pydantic
* Creating a deployment locally
* Setting up Prefect Agent
* Running the flow
* Notifications
:movie_camera: [Video](https://www.youtube.com/watch?v=QrDxPjX10iw&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
### 6. Schedules & Docker Storage with Infrastructure
* Scheduling a deployment
* Flow code storage
* Running tasks in Docker
:movie_camera: [Video](https://www.youtube.com/watch?v=psNSzqTsi-s&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
### 7. Prefect Cloud and Additional Resources
* Using Prefect Cloud instead of local Prefect
* Workspaces
* Running flows on GCP
:movie_camera: [Video](https://www.youtube.com/watch?v=gGC23ZK7lr8&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* [Prefect docs](https://docs.prefect.io/)
* [Pefect Discourse](https://discourse.prefect.io/)
* [Prefect Cloud](https://app.prefect.cloud/)
* [Prefect Slack](https://prefect-community.slack.com)
### Code repository
[Code from videos](https://github.com/discdiver/prefect-zoomcamp) (with a few minor enhancements)
### Homework
Homework can be found [here](./homework.md).
## Community notes
Did you take notes? You can share them here.
* [Blog by Marcos Torregrosa (Prefect)](https://www.n4gash.com/2023/data-engineering-zoomcamp-semana-2/)
* [Notes from Victor Padilha](https://github.com/padilha/de-zoomcamp/tree/master/week2)
* [Notes by Alain Boisvert](https://github.com/boisalai/de-zoomcamp-2023/blob/main/week2.md)
* [Notes by Candace Williams](https://github.com/teacherc/de_zoomcamp_candace2023/blob/main/week_2/week2_notes.md)
* [Notes from Xia He-Bleinagel](https://xiahe-bleinagel.com/2023/02/week-2-data-engineering-zoomcamp-notes-prefect/)
* [Notes from froukje](https://github.com/froukje/de-zoomcamp/blob/main/week_2_workflow_orchestration/notes/notes_week_02.md)
* [Notes from Balaji](https://github.com/Balajirvp/DE-Zoomcamp/blob/main/Week%202/Detailed%20Week%202%20Notes.ipynb)
* More on [Pandas vs SQL, Prefect capabilities, and testing your data](https://medium.com/@verazabeida/zoomcamp-2023-week-3-7f27bb8c483f), by Vera
* Add your notes here (above this line)

98
cohorts/2024/02-workflow-orchestration/homework.md
View File

@ -1,98 +0,0 @@
## Module 2 Homework
ATTENTION: At the end of the submission form, you will be required to include a link to your GitHub repository or other public code-hosting site. This repository should contain your code for solving the homework. If your solution includes code that is not in file format, please include these directly in the README file of your repository.
> In case you don't get one option exactly, select the closest one
For the homework, we'll be working with the _green_ taxi dataset located here:
`https://github.com/DataTalksClub/nyc-tlc-data/releases/tag/green/download`
To get a `wget`-able link, use this prefix (note that the link itself gives 404):
`https://github.com/DataTalksClub/nyc-tlc-data/releases/download/green/`
### Assignment
The goal will be to construct an ETL pipeline that loads the data, performs some transformations, and writes the data to a database (and Google Cloud!).
- Create a new pipeline, call it `green_taxi_etl`
- Add a data loader block and use Pandas to read data for the final quarter of 2020 (months `10`, `11`, `12`).
- You can use the same datatypes and date parsing methods shown in the course.
- `BONUS`: load the final three months using a for loop and `pd.concat`
- Add a transformer block and perform the following:
- Remove rows where the passenger count is equal to 0 _and_ the trip distance is equal to zero.
- Create a new column `lpep_pickup_date` by converting `lpep_pickup_datetime` to a date.
- Rename columns in Camel Case to Snake Case, e.g. `VendorID` to `vendor_id`.
- Add three assertions:
- `vendor_id` is one of the existing values in the column (currently)
- `passenger_count` is greater than 0
- `trip_distance` is greater than 0
- Using a Postgres data exporter (SQL or Python), write the dataset to a table called `green_taxi` in a schema `mage`. Replace the table if it already exists.
- Write your data as Parquet files to a bucket in GCP, partioned by `lpep_pickup_date`. Use the `pyarrow` library!
- Schedule your pipeline to run daily at 5AM UTC.
### Questions
## Question 1. Data Loading
Once the dataset is loaded, what's the shape of the data?
* 266,855 rows x 20 columns
* 544,898 rows x 18 columns
* 544,898 rows x 20 columns
* 133,744 rows x 20 columns
## Question 2. Data Transformation
Upon filtering the dataset where the passenger count is greater than 0 _and_ the trip distance is greater than zero, how many rows are left?
* 544,897 rows
* 266,855 rows
* 139,370 rows
* 266,856 rows
## Question 3. Data Transformation
Which of the following creates a new column `lpep_pickup_date` by converting `lpep_pickup_datetime` to a date?
* `data = data['lpep_pickup_datetime'].date`
* `data('lpep_pickup_date') = data['lpep_pickup_datetime'].date`
* `data['lpep_pickup_date'] = data['lpep_pickup_datetime'].dt.date`
* `data['lpep_pickup_date'] = data['lpep_pickup_datetime'].dt().date()`
## Question 4. Data Transformation
What are the existing values of `VendorID` in the dataset?
* 1, 2, or 3
* 1 or 2
* 1, 2, 3, 4
* 1
## Question 5. Data Transformation
How many columns need to be renamed to snake case?
* 3
* 6
* 2
* 4
## Question 6. Data Exporting
Once exported, how many partitions (folders) are present in Google Cloud?
* 96
* 56
* 67
* 108
## Submitting the solutions
* Form for submitting: https://courses.datatalks.club/de-zoomcamp-2024/homework/hw2
* Check the link above to see the due date
## Solution
Will be added after the due date

86
cohorts/2024/03-data-warehouse/homework.md
View File

@ -1,86 +0,0 @@
## Module 3 Homework
Solution: https://www.youtube.com/watch?v=8g_lRKaC9ro
ATTENTION: At the end of the submission form, you will be required to include a link to your GitHub repository or other public code-hosting site. This repository should contain your code for solving the homework. If your solution includes code that is not in file format (such as SQL queries or shell commands), please include these directly in the README file of your repository.
<b><u>Important Note:</b></u> <p> For this homework we will be using the 2022 Green Taxi Trip Record Parquet Files from the New York
City Taxi Data found here: </br> https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page </br>
If you are using orchestration such as Mage, Airflow or Prefect do not load the data into Big Query using the orchestrator.</br>
Stop with loading the files into a bucket. </br></br>
<u>NOTE:</u> You will need to use the PARQUET option files when creating an External Table</br>
<b>SETUP:</b></br>
Create an external table using the Green Taxi Trip Records Data for 2022. </br>
Create a table in BQ using the Green Taxi Trip Records for 2022 (do not partition or cluster this table). </br>
</p>
## Question 1:
Question 1: What is count of records for the 2022 Green Taxi Data??
- 65,623,481
- 840,402
- 1,936,423
- 253,647
## Question 2:
Write a query to count the distinct number of PULocationIDs for the entire dataset on both the tables.</br>
What is the estimated amount of data that will be read when this query is executed on the External Table and the Table?
- 0 MB for the External Table and 6.41MB for the Materialized Table
- 18.82 MB for the External Table and 47.60 MB for the Materialized Table
- 0 MB for the External Table and 0MB for the Materialized Table
- 2.14 MB for the External Table and 0MB for the Materialized Table
## Question 3:
How many records have a fare_amount of 0?
- 12,488
- 128,219
- 112
- 1,622
## Question 4:
What is the best strategy to make an optimized table in Big Query if your query will always order the results by PUlocationID and filter based on lpep_pickup_datetime? (Create a new table with this strategy)
- Cluster on lpep_pickup_datetime Partition by PUlocationID
- Partition by lpep_pickup_datetime Cluster on PUlocationID
- Partition by lpep_pickup_datetime and Partition by PUlocationID
- Cluster on by lpep_pickup_datetime and Cluster on PUlocationID
## Question 5:
Write a query to retrieve the distinct PULocationID between lpep_pickup_datetime
06/01/2022 and 06/30/2022 (inclusive)</br>
Use the materialized table you created earlier in your from clause and note the estimated bytes. Now change the table in the from clause to the partitioned table you created for question 4 and note the estimated bytes processed. What are these values? </br>
Choose the answer which most closely matches.</br>
- 22.82 MB for non-partitioned table and 647.87 MB for the partitioned table
- 12.82 MB for non-partitioned table and 1.12 MB for the partitioned table
- 5.63 MB for non-partitioned table and 0 MB for the partitioned table
- 10.31 MB for non-partitioned table and 10.31 MB for the partitioned table
## Question 6:
Where is the data stored in the External Table you created?
- Big Query
- GCP Bucket
- Big Table
- Container Registry
## Question 7:
It is best practice in Big Query to always cluster your data:
- True
- False
## (Bonus: Not worth points) Question 8:
No Points: Write a `SELECT count(*)` query FROM the materialized table you created. How many bytes does it estimate will be read? Why?
## Submitting the solutions
* Form for submitting: https://courses.datatalks.club/de-zoomcamp-2024/homework/hw3

81
cohorts/2024/04-analytics-engineering/homework.md
View File

@ -1,81 +0,0 @@
## Module 4 Homework
In this homework, we'll use the models developed during the week 4 videos and enhance the already presented dbt project using the already loaded Taxi data for fhv vehicles for year 2019 in our DWH.
This means that in this homework we use the following data [Datasets list](https://github.com/DataTalksClub/nyc-tlc-data/)
* Yellow taxi data - Years 2019 and 2020
* Green taxi data - Years 2019 and 2020
* fhv data - Year 2019.
We will use the data loaded for:
* Building a source table: `stg_fhv_tripdata`
* Building a fact table: `fact_fhv_trips`
* Create a dashboard
If you don't have access to GCP, you can do this locally using the ingested data from your Postgres database
instead. If you have access to GCP, you don't need to do it for local Postgres - only if you want to.
> **Note**: if your answer doesn't match exactly, select the closest option
### Question 1:
**What happens when we execute dbt build --vars '{'is_test_run':'true'}'**
You'll need to have completed the ["Build the first dbt models"](https://www.youtube.com/watch?v=UVI30Vxzd6c) video.
- It's the same as running *dbt build*
- It applies a _limit 100_ to all of our models
- It applies a _limit 100_ only to our staging models
- Nothing
### Question 2:
**What is the code that our CI job will run? Where is this code coming from?**
- The code that has been merged into the main branch
- The code that is behind the creation object on the dbt_cloud_pr_ schema
- The code from any development branch that has been opened based on main
- The code from the development branch we are requesting to merge to main
### Question 3 (2 points)
**What is the count of records in the model fact_fhv_trips after running all dependencies with the test run variable disabled (:false)?**
Create a staging model for the fhv data, similar to the ones made for yellow and green data. Add an additional filter for keeping only records with pickup time in year 2019.
Do not add a deduplication step. Run this models without limits (is_test_run: false).
Create a core model similar to fact trips, but selecting from stg_fhv_tripdata and joining with dim_zones.
Similar to what we've done in fact_trips, keep only records with known pickup and dropoff locations entries for pickup and dropoff locations.
Run the dbt model without limits (is_test_run: false).
- 12998722
- 22998722
- 32998722
- 42998722
### Question 4 (2 points)
**What is the service that had the most rides during the month of July 2019 month with the biggest amount of rides after building a tile for the fact_fhv_trips table and the fact_trips tile as seen in the videos?**
Create a dashboard with some tiles that you find interesting to explore the data. One tile should show the amount of trips per month, as done in the videos for fact_trips, including the fact_fhv_trips data.
- FHV
- Green
- Yellow
- FHV and Green
## Submitting the solutions
* Form for submitting: https://courses.datatalks.club/de-zoomcamp-2024/homework/hw4
Deadline: 22 February (Thursday), 22:00 CET
## Solution (To be published after deadline)
* Video: https://youtu.be/3OPggh5Rca8
* Answers:
* Question 1: It applies a _limit 100_ only to our staging models
* Question 2: The code from the development branch we are requesting to merge to main
* Question 3: 22998722
* Question 4: Yellow

98
cohorts/2024/05-batch/homework.md
View File

@ -1,98 +0,0 @@
## Week 5 Homework
In this homework we'll put what we learned about Spark in practice.
For this homework we will be using the FHV 2019-10 data found here. [FHV Data](https://github.com/DataTalksClub/nyc-tlc-data/releases/download/fhv/fhv_tripdata_2019-10.csv.gz)
### Question 1:
**Install Spark and PySpark**
- Install Spark
- Run PySpark
- Create a local spark session
- Execute spark.version.
What's the output?
> [!NOTE]
> To install PySpark follow this [guide](https://github.com/DataTalksClub/data-engineering-zoomcamp/blob/main/05-batch/setup/pyspark.md)
### Question 2:
**FHV October 2019**
Read the October 2019 FHV into a Spark Dataframe with a schema as we did in the lessons.
Repartition the Dataframe to 6 partitions and save it to parquet.
What is the average size of the Parquet (ending with .parquet extension) Files that were created (in MB)? Select the answer which most closely matches.
- 1MB
- 6MB
- 25MB
- 87MB
### Question 3:
**Count records**
How many taxi trips were there on the 15th of October?
Consider only trips that started on the 15th of October.
- 108,164
- 12,856
- 452,470
- 62,610
> [!IMPORTANT]
> Be aware of columns order when defining schema
### Question 4:
**Longest trip for each day**
What is the length of the longest trip in the dataset in hours?
- 631,152.50 Hours
- 243.44 Hours
- 7.68 Hours
- 3.32 Hours
### Question 5:
**User Interface**
Spark’s User Interface which shows the application's dashboard runs on which local port?
- 80
- 443
- 4040
- 8080
### Question 6:
**Least frequent pickup location zone**
Load the zone lookup data into a temp view in Spark</br>
[Zone Data](https://github.com/DataTalksClub/nyc-tlc-data/releases/download/misc/taxi_zone_lookup.csv)
Using the zone lookup data and the FHV October 2019 data, what is the name of the LEAST frequent pickup location Zone?</br>
- East Chelsea
- Jamaica Bay
- Union Sq
- Crown Heights North
## Submitting the solutions
- Form for submitting: https://courses.datatalks.club/de-zoomcamp-2024/homework/hw5
- Deadline: See the website

116
cohorts/2024/06-streaming/homework.md
View File

@ -1,116 +0,0 @@
## Module 6 Homework
In this homework, we're going to extend Module 5 Homework and learn about streaming with PySpark.
Ensure you have the following set up (if you had done the previous homework and the module):
- Docker
- PySpark
For this homework we will be using the files from Module 5 Homework,
- FHV 2019-10 data found here: [FHV Data](https://github.com/DataTalksClub/nyc-tlc-data/releases/download/fhv/fhv_tripdata_2019-10.csv.gz), and
- Green 2019-10 data found here: [Green Data](https://github.com/DataTalksClub/nyc-tlc-data/releases/download/green/green_tripdata_2019-10.csv.gz)
## Pre-setup
1. Extract and place the csv files in the paths under `resources` subfolder
## Spin up the containers
Set rpk alias:
```bash
alias rpk="docker exec -it redpanda-1 rpk"
```
### Question 1
Run following code to start. What is the `rpk` console version?
```bash
rpk --version
```
## Running Producer
```bash
# Run Producers for the two datasets
python producer.py --type fhv
python producer.py --type green
```
### Running Streaming Script
spark-submit script ensures installation of necessary jars before running the streaming.py
```bash
./spark-submit.sh streaming.py
```
### Question 2
**What is the most popular pickup location for FHV type taxi rides?**
- 1
- 2
- 3
- 4
## Running Consumer
```bash
# Run consumer with default settings
python3 consumer.py
# Run consumer for specific topic
python3 consumer.py --topic [topic-name]
```
### Question 4:
most popular PUlocationID for fhv trip taxis
### Question 5:
least popular DOlocationID for fhv trip taxis
## Question
```bash
rpk cluster info
rpk topic list --detailed
```
Create topic `rides_all` using the `rpk` CLI command in the terminal.
Which of these is the correct command to create topic with 1 partitions and 1 replica?
- `rpk topics creates rides_all --partitions 12 --replicas 1`
- `rpk topic rides_all --partitions 1 --replicas 1`
- `rpk topic create list rides_all --partitions 1 --replicas 1`
- `rpk topic create rides_all --partitions 1 --replicas 1`
Run the correct command in the terminal to create the topic.
### Question :
most common locationID where a taxi can drop off and pickup a passenger at the same location within a 10min threshold (windowing lesson).
<!-- scrap the above questions? -->
## Submitting the solutions
* Form for submitting: TBA
## Solution
We will publish the solution here after deadline.

35
cohorts/2024/README.md
View File

@ -1,49 +1,48 @@
## Data Engineering Zoomcamp 2024 Cohort
* [Pre-launch Q&A stream](https://www.youtube.com/watch?v=91b8u9GmqB4)
* [Launch stream with course overview](https://www.youtube.com/live/AtRhA-NfS24?si=5JzA_E8BmJjiLi8l)
* Launch stream with course overview (TODO)
* [Deadline calendar](https://docs.google.com/spreadsheets/d/e/2PACX-1vQACMLuutV5rvXg5qICuJGL-yZqIV0FBD84CxPdC5eZHf8TfzB-CJT_3Mo7U7oGVTXmSihPgQxuuoku/pubhtml)
* [Course Google calendar](https://calendar.google.com/calendar/?cid=ZXIxcjA1M3ZlYjJpcXU0dTFmaG02MzVxMG9AZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ)
* [FAQ](https://docs.google.com/document/d/19bnYs80DwuUimHM65UV3sylsCn2j1vziPOwzBwQrebw/edit?usp=sharing)
* Course Playlist: Only 2024 Live videos & homeworks (TODO)
[**Module 1: Introduction & Prerequisites**](01-docker-terraform/)
[**Module 1: Introduction & Prerequisites**](01_docker_sql/)
* [Homework](01-docker-terraform/homework.md)
* [Homework SQL](01_docker_sql/homework.md)
* [Homework Terraform](01_terraform/homework.md)
[**Module 2: Workflow Orchestration**](02_workflow_orchestration)
[**Module 2: Workflow Orchestration**](02-workflow-orchestration)
* [Homework](02-workflow-orchestration/homework.md)
* [Homework](02_workflow_orchestration/homework.md)
* Office hours
[**Workshop 1: Data Ingestion**](workshops/dlt.md)
[**Workshop: Data Loading**]
* Workshop with dlt
* [Homework](workshops/dlt.md)
Workshop with dlt (TBA)
[**Module 3: Data Warehouse**](03-data-warehouse)
[**Module 3: Data Warehouse**](03_data_warehouse)
* [Homework](03-data-warehouse/homework.md)
* [Homework](03_data_warehouse/homework.md)
[**Module 4: Analytics Engineering**](04-analytics-engineering/)
[**Module 4: Analytics Engineering**](04_analytics_engineering/)
* [Homework](04-analytics-engineering/homework.md)
* [Homework](04_analytics_engineering/homework.md)
[**Module 5: Batch processing**](05-batch/)
[**Module 5: Batch processing**](05_batch_processing/)
* [Homework](05-batch/homework.md)
* [Homework](05_batch_processing/homework.md)
[**Module 6: Stream Processing**](06-streaming)
[**Module 6: Stream Processing**](06_stream_processing)
* [Homework](06-streaming/homework.md)
* [Homework](06_stream_processing/homework.md)
[**Project**](project.md)
[**Module 7, 8 & 9: Project**](project.md)
More information [here](project.md)

2
cohorts/2024/project.md
View File

@ -58,7 +58,7 @@ Project feedback: TBA ("project-02" sheet)
### Evaluation criteria
See [here](../../projects/README.md)
See [here](../../week_7_project/README.md)
### Misc

16
cohorts/2024/01-docker-terraform/homework.md → cohorts/2024/week_1_basics_n_setup/homework.md
View File

@ -1,7 +1,5 @@
## Module 1 Homework
ATTENTION: At the very end of the submission form, you will be required to include a link to your GitHub repository or other public code-hosting site. This repository should contain your code for solving the homework. If your solution includes code that is not in file format (such as SQL queries or shell commands), please include these directly in the README file of your repository.
## Docker & SQL
In this homework we'll prepare the environment
@ -68,20 +66,18 @@ Remember that `lpep_pickup_datetime` and `lpep_dropoff_datetime` columns are in
- 15859
- 89009
## Question 4. Longest trip for each day
## Question 4. Largest trip for each day
Which was the pick up day with the longest trip distance?
Which was the pick up day with the largest trip distance
Use the pick up time for your calculations.
Tip: For every trip on a single day, we only care about the trip with the longest distance.
- 2019-09-18
- 2019-09-16
- 2019-09-26
- 2019-09-21
## Question 5. Three biggest pick up Boroughs
## Question 5. The number of passengers
Consider lpep_pickup_datetime in '2019-09-18' and ignoring Borough has Unknown
@ -113,7 +109,7 @@ In this section homework we'll prepare the environment by creating resources in
In your VM on GCP/Laptop/GitHub Codespace install Terraform.
Copy the files from the course repo
[here](https://github.com/DataTalksClub/data-engineering-zoomcamp/tree/main/01-docker-terraform/1_terraform_gcp/terraform) to your VM/Laptop/GitHub Codespace.
[here](https://github.com/DataTalksClub/data-engineering-zoomcamp/tree/main/week_1_basics_n_setup/1_terraform_gcp/terraform) to your VM/Laptop/GitHub Codespace.
Modify the files as necessary to create a GCP Bucket and Big Query Dataset.
@ -131,7 +127,7 @@ Paste the output of this command into the homework submission form.
## Submitting the solutions
* Form for submitting: https://courses.datatalks.club/de-zoomcamp-2024/homework/hw01
* Form for submitting:
* You can submit your homework multiple times. In this case, only the last submission will be used.
Deadline: 29 January, 23:00 CET
Deadline:

133
cohorts/2024/workshops/dlt.md
View File

@ -1,133 +0,0 @@
# Data ingestion with dlt
​In this hands-on workshop, we’ll learn how to build data ingestion pipelines.
​We’ll cover the following steps:
* ​Extracting data from APIs, or files.
* ​Normalizing and loading data
* ​Incremental loading
​By the end of this workshop, you’ll be able to write data pipelines like a senior data engineer: Quickly, concisely, scalable, and self-maintaining.
Video: https://www.youtube.com/live/oLXhBM7nf2Q
---
# Navigation
* [Workshop content](dlt_resources/data_ingestion_workshop.md)
* [Workshop notebook](dlt_resources/workshop.ipynb)
* [Homework starter notebook](dlt_resources/homework_starter.ipynb)
# Resources
- Website and community: Visit our [docs](https://dlthub.com/docs/intro), discuss on our slack (Link at top of docs).
- Course colab: [Notebook](https://colab.research.google.com/drive/1kLyD3AL-tYf_HqCXYnA3ZLwHGpzbLmoj#scrollTo=5aPjk0O3S_Ag&forceEdit=true&sandboxMode=true).
- dlthub [community Slack](https://dlthub.com/community).
---
# Teacher
Welcome to the data talks club data engineering zoomcamp, the data ingestion workshop.
- My name is [Adrian](https://www.linkedin.com/in/data-team/), and I work in the data field since 2012
- I built many data warehouses some lakes, and a few data teams
- 10 years into my career I started working on dlt “data load tool”, which is an open source library to enable data engineers to build faster and better.
- I started working on dlt because data engineering is one of the few areas of software engineering where we do not have developer tools to do our work.
- Building better pipelines would require more code re-use - we cannot all just build perfect pipelines from scratch every time.
- And so dlt was born, a library that automates the tedious part of data ingestion: Loading, schema management, data type detection, scalability, self healing, scalable extraction… you get the idea - essentially a data engineer’s “one stop shop” for best practice data pipelining.
- Due to its **simplicity** of use, dlt enables **laymen** to
- Build pipelines 5-10x faster than without it
- Build self healing, self maintaining pipelines with all the best practices of data engineers. Automating schema changes removes the bulk of maintenance efforts.
- Govern your pipelines with schema evolution alerts and data contracts.
- and generally develop pipelines like a senior, commercial data engineer.
---
# Course
You can find the course file [here](./dlt_resources/data_ingestion_workshop.md)
The course has 3 parts
- [Extraction Section](./dlt_resources/data_ingestion_workshop.md#extracting-data): In this section we will learn about scalable extraction
- [Normalisation Section](./dlt_resources/data_ingestion_workshop.md#normalisation): In this section we will learn to prepare data for loading
- [Loading Section](./dlt_resources/data_ingestion_workshop.md#incremental-loading)): Here we will learn about incremental loading modes
---
# Homework
The [linked colab notebook](https://colab.research.google.com/drive/1Te-AT0lfh0GpChg1Rbd0ByEKOHYtWXfm#scrollTo=wLF4iXf-NR7t&forceEdit=true&sandboxMode=true) offers a few exercises to practice what you learned today.
#### Question 1: What is the sum of the outputs of the generator for limit = 5?
- **A**: 10.23433234744176
- **B**: 7.892332347441762
- **C**: 8.382332347441762
- **D**: 9.123332347441762
#### Question 2: What is the 13th number yielded by the generator?
- **A**: 4.236551275463989
- **B**: 3.605551275463989
- **C**: 2.345551275463989
- **D**: 5.678551275463989
#### Question 3: Append the 2 generators. After correctly appending the data, calculate the sum of all ages of people.
- **A**: 353
- **B**: 365
- **C**: 378
- **D**: 390
#### Question 4: Merge the 2 generators using the ID column. Calculate the sum of ages of all the people loaded as described above.
- **A**: 215
- **B**: 266
- **C**: 241
- **D**: 258
Submit the solution here: https://courses.datatalks.club/de-zoomcamp-2024/homework/workshop1
---
# Next steps
As you are learning the various concepts of data engineering,
consider creating a portfolio project that will further your own knowledge.
By demonstrating the ability to deliver end to end, you will have an easier time finding your first role.
This will help regardless of whether your hiring manager reviews your project, largely because you will have a better
understanding and will be able to talk the talk.
Here are some example projects that others did with dlt:
- Serverless dlt-dbt on cloud functions: [Article](https://docs.getdbt.com/blog/serverless-dlt-dbt-stack)
- Bird finder: [Part 1](https://publish.obsidian.md/lough-on-data/blogs/bird-finder-via-dlt-i), [Part 2](https://publish.obsidian.md/lough-on-data/blogs/bird-finder-via-dlt-ii)
- Event ingestion on GCP: [Article and repo](https://dlthub.com/docs/blog/streaming-pubsub-json-gcp)
- Event ingestion on AWS: [Article and repo](https://dlthub.com/docs/blog/dlt-aws-taktile-blog)
- Or see one of the many demos created by our working students: [Hacker news](https://dlthub.com/docs/blog/hacker-news-gpt-4-dashboard-demo),
[GA4 events](https://dlthub.com/docs/blog/ga4-internal-dashboard-demo),
[an E-Commerce](https://dlthub.com/docs/blog/postgresql-bigquery-metabase-demo),
[google sheets](https://dlthub.com/docs/blog/google-sheets-to-data-warehouse-pipeline),
[Motherduck](https://dlthub.com/docs/blog/dlt-motherduck-demo),
[MongoDB + Holistics](https://dlthub.com/docs/blog/MongoDB-dlt-Holistics),
[Deepnote](https://dlthub.com/docs/blog/deepnote-women-wellness-violence-tends),
[Prefect](https://dlthub.com/docs/blog/dlt-prefect),
[PowerBI vs GoodData vs Metabase](https://dlthub.com/docs/blog/semantic-modeling-tools-comparison),
[Dagster](https://dlthub.com/docs/blog/dlt-dagster),
[Ingesting events via gcp webhooks](https://dlthub.com/docs/blog/dlt-webhooks-on-cloud-functions-for-event-capture),
[SAP to snowflake replication](https://dlthub.com/docs/blog/sap-hana-to-snowflake-demo-blog),
[Read emails and send sumamry to slack with AI and Kestra](https://dlthub.com/docs/blog/dlt-kestra-demo-blog),
[Mode +dlt capabilities](https://dlthub.com/docs/blog/dlt-mode-blog),
[dbt on cloud functions](https://dlthub.com/docs/blog/dlt-dbt-runner-on-cloud-functions)
- If you want to use dlt in your project, [check this list of public APIs](https://dlthub.com/docs/blog/practice-api-sources)
If you create a personal project, consider submitting it to our blog - we will be happy to showcase it. Just drop us a line in the dlt slack.
**And don't forget, if you like dlt**
- **Give us a [GitHub Star!](https://github.com/dlt-hub/dlt)**
- **Join our [Slack community](https://dlthub.com/community)**
# Notes
* Add your notes here

582
cohorts/2024/workshops/dlt_resources/data_ingestion_workshop.md
View File

@ -1,582 +0,0 @@
# Intro
What is data loading, or data ingestion?
Data ingestion is the process of extracting data from a producer, transporting it to a convenient environment, and preparing it for usage by normalising it, sometimes cleaning, and adding metadata.
### “A wild dataset magically appears!”
In many data science teams, data magically appears - because the engineer loads it.
- Sometimes the format in which it appears is structured, and with explicit schema
- In that case, they can go straight to using it; Examples: Parquet, Avro, or table in a db,
- Sometimes the format is weakly typed and without explicit schema, such as csv, json
- in which case some extra normalisation or cleaning might be needed before usage
> 💡 **What is a schema?** The schema specifies the expected format and structure of data within a document or data store, defining the allowed keys, their data types, and any constraints or relationships.
### Be the magician! 😎
Since you are here to learn about data engineering, you will be the one making datasets magically appear.
Here’s what you need to learn to build pipelines
- Extracting data
- Normalising, cleaning, adding metadata such as schema and types
- and Incremental loading, which is vital for fast, cost effective data refreshes.
### What else does a data engineer do? What are we not learning, and what are we learning?
- It might seem simplistic, but in fact a data engineer’s main goal is to ensure data flows from source systems to analytical destinations.
- So besides building pipelines, running pipelines and fixing pipelines, a data engineer may also focus on optimising data storage, ensuring data quality and integrity, implementing effective data governance practices, and continuously refining data architecture to meet the evolving needs of the organisation.
- Ultimately, a data engineer's role extends beyond the mechanical aspects of pipeline development, encompassing the strategic management and enhancement of the entire data lifecycle.
- This workshop focuses on building robust, scalable, self maintaining pipelines, with built in governance - in other words, best practices applied.
# Extracting data
### The considerations of extracting data
In this section we will learn about extracting data from source systems, and what to care about when doing so.
Most data is stored behind an API
- Sometimes that’s a RESTful api for some business application, returning records of data.
- Sometimes the API returns a secure file path to something like a json or parquet file in a bucket that enables you to grab the data in bulk,
- Sometimes the API is something else (mongo, sql, other databases or applications) and will generally return records as JSON - the most common interchange format.
As an engineer, you will need to build pipelines that “just work”.
So here’s what you need to consider on extraction, to prevent the pipelines from breaking, and to keep them running smoothly.
- Hardware limits: During this course we will cover how to navigate the challenges of managing memory.
- Network limits: Sometimes networks can fail. We can’t fix what could go wrong but we can retry network jobs until they succeed. For example, dlt library offers a requests “replacement” that has built in retries. [Docs](https://dlthub.com/docs/reference/performance#using-the-built-in-requests-client). We won’t focus on this during the course but you can read the docs on your own.
- Source api limits: Each source might have some limits such as how many requests you can do per second. We would call these “rate limits”. Read each source’s docs carefully to understand how to navigate these obstacles. You can find some examples of how to wait for rate limits in our verified sources repositories
- examples: [Zendesk](https://developer.zendesk.com/api-reference/introduction/rate-limits/), [Shopify](https://shopify.dev/docs/api/usage/rate-limits)
### Extracting data without hitting hardware limits
What kind of limits could you hit on your machine? In the case of data extraction, the only limits are memory and storage. This refers to the RAM or virtual memory, and the disk, or physical storage.
### **Managing memory.**
- Many data pipelines run on serverless functions or on orchestrators that delegate the workloads to clusters of small workers.
- These systems have a small memory or share it between multiple workers - so filling the memory is BAAAD: It might lead to not only your pipeline crashing, but crashing the entire container or machine that might be shared with other worker processes, taking them down too.
- The same can be said about disk - in most cases your disk is sufficient, but in some cases it’s not. For those cases, mounting an external drive mapped to a storage bucket is the way to go. Airflow for example supports a “data” folder that is used just like a local folder but can be mapped to a bucket for unlimited capacity.
### So how do we avoid filling the memory?
- We often do not know the volume of data upfront
- And we cannot scale dynamically or infinitely on hardware during runtime
- So the answer is: Control the max memory you use
### Control the max memory used by streaming the data
Streaming here refers to processing the data event by event or chunk by chunk instead of doing bulk operations.
Let’s look at some classic examples of streaming where data is transferred chunk by chunk or event by event
- Between an audio broadcaster and an in-browser audio player
- Between a server and a local video player
- Between a smart home device or IoT device and your phone
- between google maps and your navigation app
- Between instagram live and your followers
What do data engineers do? We usually stream the data between buffers, such as
- from API to local file
- from webhooks to event queues
- from event queue (Kafka, SQS) to Bucket
### Streaming in python via generators
Let’s focus on how we build most data pipelines:
- To process data in a stream in python, we use generators, which are functions that can return multiple times - by allowing multiple returns, the data can be released as it’s produced, as stream, instead of returning it all at once as a batch.
Take the following theoretical example:
- We search twitter for “cat pictures”. We do not know how many pictures will be returned - maybe 10, maybe 10.000.000. Will they fit in memory? Who knows.
- So to grab this data without running out of memory, we would use a python generator.
- What’s a generator? In simple words, it’s a function that can return multiple times. Here’s an example of a regular function, and how that function looks if written as a generator.
### Generator examples:
Let’s look at a regular returning function, and how we can re-write it as a generator.
**Regular function collects data in memory.** Here you can see how data is collected row by row in a list called `data`before it is returned. This will break if we have more data than memory.
```python
def search_twitter(query):
data = []
for row in paginated_get(query):
data.append(row)
return data
# Collect all the cat picture data
for row in search_twitter("cat pictures"):
# Once collected,
# print row by row
print(row)
```
When calling `for row in search_twitter("cat pictures"):` all the data must first be downloaded before the first record is returned
Let’s see how we could rewrite this as a generator.
**Generator for streaming the data.** The memory usage here is minimal.
As you can see, in the modified function, we yield each row as we get the data, without collecting it into memory. We can then run this generator and handle the data item by item.
```python
def search_twitter(query):
for row in paginated_get(query):
yield row
# Get one row at a time
for row in extract_data("cat pictures"):
# print the row
print(row)
# do something with the row such as cleaning it and writing it to a buffer
# continue requesting and printing data
```
When calling `for row in extract_data("cat pictures"):` the function only runs until the first data item is yielded, before printing - so we do not need to wait long for the first value. It will then continue until there is no more data to get.
If we wanted to get all the values at once from a generator instead of one by one, we would need to first “run” the generator and collect the data. For example, if we wanted to get all the data in memory we could do `data = list(extract_data("cat pictures"))` which would run the generator and collect all the data in a list before continuing.
## 3 Extraction examples:
### Example 1: Grabbing data from an api
> 💡 This is the bread and butter of data engineers pulling data, so follow along in the colab or in your local setup.
For these purposes we created an api that can serve the data you are already familiar with, the NYC taxi dataset.
The api documentation is as follows:
- There are a limited nr of records behind the api
- The data can be requested page by page, each page containing 1000 records
- If we request a page with no data, we will get a successful response with no data
- so this means that when we get an empty page, we know there is no more data and we can stop requesting pages - this is a common way to paginate but not the only one - each api may be different.
- details:
- method: get
- url: `https://us-central1-dlthub-analytics.cloudfunctions.net/data_engineering_zoomcamp_api`
- parameters: `page` integer. Represents the page number you are requesting. Defaults to 1.
So how do we design our requester?
- We need to request page by page until we get no more data. At this point, we do not know how much data is behind the api.
- It could be 1000 records or it could be 10GB of records. So let’s grab the data with a generator to avoid having to fit an undetermined amount of data into ram.
In this approach to grabbing data from apis, we have pros and cons:
- Pros: **Easy memory management** thanks to api returning events/pages
- Cons: **Low throughput**, due to the data transfer being constrained via an API.
```bash
import requests
BASE_API_URL = "https://us-central1-dlthub-analytics.cloudfunctions.net/data_engineering_zoomcamp_api"
# I call this a paginated getter
# as it's a function that gets data
# and also paginates until there is no more data
# by yielding pages, we "microbatch", which speeds up downstream processing
def paginated_getter():
page_number = 1
while True:
# Set the query parameters
params = {'page': page_number}
# Make the GET request to the API
response = requests.get(BASE_API_URL, params=params)
response.raise_for_status() # Raise an HTTPError for bad responses
page_json = response.json()
print(f'got page number {page_number} with {len(page_json)} records')
# if the page has no records, stop iterating
if page_json:
yield page_json
page_number += 1
else:
# No more data, break the loop
break
if __name__ == '__main__':
# Use the generator to iterate over pages
for page_data in paginated_getter():
# Process each page as needed
print(page_data)
```
### Example 2: Grabbing the same data from file - simple download
> 💡 This part is demonstrative, so you do not need to follow along; just pay attention.
- Why am I showing you this? so when you do this in the future, you will remember there is a best practice you can apply for scalability.
Some apis respond with files instead of pages of data. The reason for this is simple: Throughput and cost. A restful api that returns data has to read the data from storage and process and return it to you by some logic - If this data is large, this costs time, money and creates a bottleneck.
A better way is to offer the data as files that someone can download from storage directly, without going through the restful api layer. This is common for apis that offer large volumes of data, such as ad impressions data.
In this example, we grab exactly the same data as we did in the API example above, but now we get it from the underlying file instead of going through the API.
- Pros: **High throughput**
- Cons: **Memory** is used to hold all the data
This is how the code could look. As you can see in this case our `data`and `parsed_data` variables hold the entire file data in memory before returning it. Not great.
```python
import requests
import json
url = "https://storage.googleapis.com/dtc_zoomcamp_api/yellow_tripdata_2009-06.jsonl"
def download_and_read_jsonl(url):
response = requests.get(url)
response.raise_for_status() # Raise an HTTPError for bad responses
data = response.text.splitlines()
parsed_data = [json.loads(line) for line in data]
return parsed_data
downloaded_data = download_and_read_jsonl(url)
if downloaded_data:
# Process or print the downloaded data as needed
print(downloaded_data[:5]) # Print the first 5 entries as an example
```
### Example 3: Same file, streaming download
> 💡 This is the bread and butter of data engineers pulling data, so follow along in the colab
Ok, downloading files is simple, but what if we want to do a stream download?
That’s possible too - in effect giving us the best of both worlds. In this case we prepared a jsonl file which is already split into lines making our code simple. But json (not jsonl) files could also be downloaded in this fashion, for example using the `ijson` library.
What are the pros and cons of this method of grabbing data?
Pros: **High throughput, easy memory management,** because we are downloading a file
Cons: **Difficult to do for columnar file formats**, as entire blocks need to be downloaded before they can be deserialised to rows. Sometimes, the code is complex too.
Here’s what the code looks like - in a jsonl file each line is a json document, or a “row” of data, so we yield them as they get downloaded. This allows us to download one row and process it before getting the next row.
```bash
import requests
import json
def download_and_yield_rows(url):
response = requests.get(url, stream=True)
response.raise_for_status() # Raise an HTTPError for bad responses
for line in response.iter_lines():
if line:
yield json.loads(line)
# Replace the URL with your actual URL
url = "https://storage.googleapis.com/dtc_zoomcamp_api/yellow_tripdata_2009-06.jsonl"
# Use the generator to iterate over rows with minimal memory usage
for row in download_and_yield_rows(url):
# Process each row as needed
print(row)
```
In the colab notebook you can also find a code snippet to load the data - but we will load some data later in the course and you can explore the colab on your own after the course.
What is worth keeping in mind at this point is that our loader library that we will use later, `dlt`or data load tool, will respect the streaming concept of the generator and will process it in an efficient way keeping memory usage low and using parallelism where possible.
Let’s move over to the Colab notebook and run examples 2 and 3, compare them, and finally load examples 1 and 3 to DuckDB
# Normalising data
You often hear that data people spend most of their time “cleaning” data. What does this mean?
Let’s look granularly into what people consider data cleaning.
Usually we have 2 parts:
- Normalising data without changing its meaning,
- and filtering data for a use case, which changes its meaning.
### Part of what we often call data cleaning is just metadata work:
- Add types (string to number, string to timestamp, etc)
- Rename columns: Ensure column names follow a supported standard downstream - such as no strange characters in the names.
- Flatten nested dictionaries: Bring nested dictionary values into the top dictionary row
- Unnest lists or arrays into child tables: Arrays or lists cannot be flattened into their parent record, so if we want flat data we need to break them out into separate tables.
- We will look at a practical example next, as these concepts can be difficult to visualise from text.
### **Why prepare data? why not use json as is?**
- We do not easily know what is inside a json document due to lack of schema
- Types are not enforced between rows of json - we could have one record where age is `25`and another where age is `twenty five` , and another where it’s `25.00`. Or in some systems, you might have a dictionary for a single record, but a list of dicts for multiple records. This could easily lead to applications downstream breaking.
- We cannot just use json data easily, for example we would need to convert strings to time if we want to do a daily aggregation.
- Reading json loads more data into memory, as the whole document is scanned - while in parquet or databases we can scan a single column of a document. This causes costs and slowness.
- Json is not fast to aggregate - columnar formats are.
- Json is not fast to search.
- Basically json is designed as a "lowest common denominator format" for "interchange" / data transfer and is unsuitable for direct analytical usage.
### Practical example
> 💡 This is the bread and butter of data engineers pulling data, so follow along in the colab notebook.
In the case of the NY taxi rides data, the dataset is quite clean - so let’s instead use a small example of more complex data. Let’s assume we know some information about passengers and stops.
For this example we modified the dataset as follows
- We added nested dictionaries
```json
"coordinates": {
"start": {
"lon": -73.787442,
"lat": 40.641525
},
```
- We added nested lists
```json
"passengers": [
{"name": "John", "rating": 4.9},
{"name": "Jack", "rating": 3.9}
],
```
- We added a record hash that gives us an unique id for the record, for easy identification
```json
"record_hash": "b00361a396177a9cb410ff61f20015ad",
```
We want to load this data to a database. How do we want to clean the data?
- We want to flatten dictionaries into the base row
- We want to flatten lists into a separate table
- We want to convert time strings into time type
```python
data = [
{
"vendor_name": "VTS",
"record_hash": "b00361a396177a9cb410ff61f20015ad",
"time": {
"pickup": "2009-06-14 23:23:00",
"dropoff": "2009-06-14 23:48:00"
},
"Trip_Distance": 17.52,
"coordinates": {
"start": {
"lon": -73.787442,
"lat": 40.641525
},
"end": {
"lon": -73.980072,
"lat": 40.742963
}
},
"Rate_Code": None,
"store_and_forward": None,
"Payment": {
"type": "Credit",
"amt": 20.5,
"surcharge": 0,
"mta_tax": None,
"tip": 9,
"tolls": 4.15,
"status": "booked"
},
"Passenger_Count": 2,
"passengers": [
{"name": "John", "rating": 4.9},
{"name": "Jack", "rating": 3.9}
],
"Stops": [
{"lon": -73.6, "lat": 40.6},
{"lon": -73.5, "lat": 40.5}
]
},
]
```
Now let’s normalise this data.
## Introducing dlt
dlt is a python library created for the purpose of assisting data engineers to build simpler, faster and more robust pipelines with minimal effort.
You can think of dlt as a loading tool that implements the best practices of data pipelines enabling you to just “use” those best practices in your own pipelines, in a declarative way.
This enables you to stop reinventing the flat tyre, and leverage dlt to build pipelines much faster than if you did everything from scratch.
dlt automates much of the tedious work a data engineer would do, and does it in a way that is robust. dlt can handle things like:
- Schema: Inferring and evolving schema, alerting changes, using schemas as data contracts.
- Typing data, flattening structures, renaming columns to fit database standards. In our example we will pass the “data” you can see above and see it normalised.
- Processing a stream of events/rows without filling memory. This includes extraction from generators.
- Loading to a variety of dbs or file formats.
Let’s use it to load our nested json to duckdb:
Here’s how you would do that on your local machine. I will walk you through before showing you in colab as well.
First, install dlt
```bash
# Make sure you are using Python 3.8-3.11 and have pip installed
# spin up a venv
python -m venv ./env
source ./env/bin/activate
# pip install
pip install dlt[duckdb]
```
Next, grab your data from above and run this snippet
- here we define a pipeline, which is a connection to a destination
- and we run the pipeline, printing the outcome
```python
# define the connection to load to.
# We now use duckdb, but you can switch to Bigquery later
pipeline = dlt.pipeline(pipeline_name="taxi_data",
destination='duckdb',
dataset_name='taxi_rides')
# run the pipeline with default settings, and capture the outcome
info = pipeline.run(data,
table_name="users",
write_disposition="replace")
# show the outcome
print(info)
```
If you are running dlt locally you can use the built in streamlit app by running the cli command with the pipeline name we chose above.
```bash
dlt pipeline taxi_data show
```
Or explore the data in the linked colab notebook. I’ll switch to it now to show you the data.
# Incremental loading
Incremental loading means that as we update our datasets with the new data, we would only load the new data, as opposed to making a full copy of a source’s data all over again and replacing the old version.
By loading incrementally, our pipelines run faster and cheaper.
- Incremental loading goes hand in hand with incremental extraction and state, two concepts which we will not delve into during this workshop
- `State` is information that keeps track of what was loaded, to know what else remains to be loaded. dlt stores the state at the destination in a separate table.
- Incremental extraction refers to only requesting the increment of data that we need, and not more. This is tightly connected to the state to determine the exact chunk that needs to be extracted and loaded.
- You can learn more about incremental extraction and state by reading the dlt docs on how to do it.
### dlt currently supports 2 ways of loading incrementally:
1. Append:
- We can use this for immutable or stateless events (data that doesn’t change), such as taxi rides - For example, every day there are new rides, and we could load the new ones only instead of the entire history.
- We could also use this to load different versions of stateful data, for example for creating a “slowly changing dimension” table for auditing changes. For example, if we load a list of cars and their colors every day, and one day one car changes color, we need both sets of data to be able to discern that a change happened.
2. Merge:
- We can use this to update data that changes.
- For example, a taxi ride could have a payment status, which is originally “booked” but could later be changed into “paid”, “rejected” or “cancelled”
Here is how you can think about which method to use:
![Incremental Loading](./incremental_loading.png)
* If you want to keep track of when changes occur in stateful data (slowly changing dimension) then you will need to append the data
### Let’s do a merge example together:
> 💡 This is the bread and butter of data engineers pulling data, so follow along.
- In our previous example, the payment status changed from "booked" to “cancelled”. Perhaps Jack likes to fraud taxis and that explains his low rating. Besides the ride status change, he also got his rating lowered further.
- The merge operation replaces an old record with a new one based on a key. The key could consist of multiple fields or a single unique id. We will use record hash that we created for simplicity. If you do not have a unique key, you could create one deterministically out of several fields, such as by concatenating the data and hashing it.
- A merge operation replaces rows, it does not update them. If you want to update only parts of a row, you would have to load the new data by appending it and doing a custom transformation to combine the old and new data.
In this example, the score of the 2 drivers got lowered and we need to update the values. We do it by using merge write disposition, replacing the records identified by `record hash` present in the new data.
```python
data = [
{
"vendor_name": "VTS",
"record_hash": "b00361a396177a9cb410ff61f20015ad",
"time": {
"pickup": "2009-06-14 23:23:00",
"dropoff": "2009-06-14 23:48:00"
},
"Trip_Distance": 17.52,
"coordinates": {
"start": {
"lon": -73.787442,
"lat": 40.641525
},
"end": {
"lon": -73.980072,
"lat": 40.742963
}
},
"Rate_Code": None,
"store_and_forward": None,
"Payment": {
"type": "Credit",
"amt": 20.5,
"surcharge": 0,
"mta_tax": None,
"tip": 9,
"tolls": 4.15,
"status": "cancelled"
},
"Passenger_Count": 2,
"passengers": [
{"name": "John", "rating": 4.4},
{"name": "Jack", "rating": 3.6}
],
"Stops": [
{"lon": -73.6, "lat": 40.6},
{"lon": -73.5, "lat": 40.5}
]
},
]
# define the connection to load to.
# We now use duckdb, but you can switch to Bigquery later
pipeline = dlt.pipeline(destination='duckdb', dataset_name='taxi_rides')
# run the pipeline with default settings, and capture the outcome
info = pipeline.run(data,
table_name="users",
write_disposition="merge",
merge_key="record_hash")
# show the outcome
print(info)
```
As you can see in your notebook, the payment status and Jack’s rating were updated after running the code.
### What’s next?
- You could change the destination to parquet + local file system or storage bucket. See the colab bonus section.
- You could change the destination to BigQuery. Destination & credential setup docs: https://dlthub.com/docs/dlt-ecosystem/destinations/, https://dlthub.com/docs/walkthroughs/add_credentials
or See the colab bonus section.
- You could use a decorator to convert the generator into a customised dlt resource: https://dlthub.com/docs/general-usage/resource
- You can deep dive into building more complex pipelines by following the guides:
- https://dlthub.com/docs/walkthroughs
- https://dlthub.com/docs/build-a-pipeline-tutorial
- You can join our [Slack community](https://dlthub.com/community) and engage with us there.

1239
cohorts/2024/workshops/dlt_resources/homework_solution.ipynb
View File

File diff suppressed because it is too large Load Diff

233
cohorts/2024/workshops/dlt_resources/homework_starter.ipynb
View File

@ -1,233 +0,0 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# **Homework**: Data talks club data engineering zoomcamp Data loading workshop\n",
"\n",
"Hello folks, let's practice what we learned - Loading data with the best practices of data engineering.\n",
"\n",
"Here are the exercises we will do\n",
"\n",
"\n"
],
"metadata": {
"id": "mrTFv5nPClXh"
}
},
{
"cell_type": "markdown",
"source": [
"# 1. Use a generator\n",
"\n",
"Remember the concept of generator? Let's practice using them to futher our understanding of how they work.\n",
"\n",
"Let's define a generator and then run it as practice.\n",
"\n",
"**Answer the following questions:**\n",
"\n",
"- **Question 1: What is the sum of the outputs of the generator for limit = 5?**\n",
"- **Question 2: What is the 13th number yielded**\n",
"\n",
"I suggest practicing these questions without GPT as the purpose is to further your learning."
],
"metadata": {
"id": "wLF4iXf-NR7t"
}
},
{
"cell_type": "code",
"source": [
"def square_root_generator(limit):\n",
" n = 1\n",
" while n <= limit:\n",
" yield n ** 0.5\n",
" n += 1\n",
"\n",
"# Example usage:\n",
"limit = 5\n",
"generator = square_root_generator(limit)\n",
"\n",
"for sqrt_value in generator:\n",
" print(sqrt_value)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "wLng-bDJN4jf",
"outputId": "547683cb-5f56-4815-a903-d0d9578eb1f9"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"1.0\n",
"1.4142135623730951\n",
"1.7320508075688772\n",
"2.0\n",
"2.23606797749979\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [],
"metadata": {
"id": "xbe3q55zN43j"
}
},
{
"cell_type": "markdown",
"source": [
"# 2. Append a generator to a table with existing data\n",
"\n",
"\n",
"Below you have 2 generators. You will be tasked to load them to duckdb and answer some questions from the data\n",
"\n",
"1. Load the first generator and calculate the sum of ages of all people. Make sure to only load it once.\n",
"2. Append the second generator to the same table as the first.\n",
"3. **After correctly appending the data, calculate the sum of all ages of people.**\n",
"\n",
"\n"
],
"metadata": {
"id": "vjWhILzGJMpK"
}
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "2MoaQcdLBEk6",
"outputId": "d2b93dc1-d83f-44ea-aeff-fdf51d75f7aa"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{'ID': 1, 'Name': 'Person_1', 'Age': 26, 'City': 'City_A'}\n",
"{'ID': 2, 'Name': 'Person_2', 'Age': 27, 'City': 'City_A'}\n",
"{'ID': 3, 'Name': 'Person_3', 'Age': 28, 'City': 'City_A'}\n",
"{'ID': 4, 'Name': 'Person_4', 'Age': 29, 'City': 'City_A'}\n",
"{'ID': 5, 'Name': 'Person_5', 'Age': 30, 'City': 'City_A'}\n",
"{'ID': 3, 'Name': 'Person_3', 'Age': 33, 'City': 'City_B', 'Occupation': 'Job_3'}\n",
"{'ID': 4, 'Name': 'Person_4', 'Age': 34, 'City': 'City_B', 'Occupation': 'Job_4'}\n",
"{'ID': 5, 'Name': 'Person_5', 'Age': 35, 'City': 'City_B', 'Occupation': 'Job_5'}\n",
"{'ID': 6, 'Name': 'Person_6', 'Age': 36, 'City': 'City_B', 'Occupation': 'Job_6'}\n",
"{'ID': 7, 'Name': 'Person_7', 'Age': 37, 'City': 'City_B', 'Occupation': 'Job_7'}\n",
"{'ID': 8, 'Name': 'Person_8', 'Age': 38, 'City': 'City_B', 'Occupation': 'Job_8'}\n"
]
}
],
"source": [
"def people_1():\n",
" for i in range(1, 6):\n",
" yield {\"ID\": i, \"Name\": f\"Person_{i}\", \"Age\": 25 + i, \"City\": \"City_A\"}\n",
"\n",
"for person in people_1():\n",
" print(person)\n",
"\n",
"\n",
"def people_2():\n",
" for i in range(3, 9):\n",
" yield {\"ID\": i, \"Name\": f\"Person_{i}\", \"Age\": 30 + i, \"City\": \"City_B\", \"Occupation\": f\"Job_{i}\"}\n",
"\n",
"\n",
"for person in people_2():\n",
" print(person)\n"
]
},
{
"cell_type": "markdown",
"source": [],
"metadata": {
"id": "vtdTIm4fvQCN"
}
},
{
"cell_type": "markdown",
"source": [
"# 3. Merge a generator\n",
"\n",
"Re-use the generators from Exercise 2.\n",
"\n",
"A table's primary key needs to be created from the start, so load your data to a new table with primary key ID.\n",
"\n",
"Load your first generator first, and then load the second one with merge. Since they have overlapping IDs, some of the records from the first load should be replaced by the ones from the second load.\n",
"\n",
"After loading, you should have a total of 8 records, and ID 3 should have age 33.\n",
"\n",
"Question: **Calculate the sum of ages of all the people loaded as described above.**\n"
],
"metadata": {
"id": "pY4cFAWOSwN1"
}
},
{
"cell_type": "markdown",
"source": [
"# Solution: First make sure that the following modules are installed:"
],
"metadata": {
"id": "kKB2GTB9oVjr"
}
},
{
"cell_type": "code",
"source": [
"#Install the dependencies\n",
"%%capture\n",
"!pip install dlt[duckdb]"
],
"metadata": {
"id": "xTVvtyqrfVNq"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# to do: homework :)"
],
"metadata": {
"id": "a2-PRBAkGC2K"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"Questions? difficulties? We are here to help.\n",
"- DTC data engineering course channel: https://datatalks-club.slack.com/archives/C01FABYF2RG\n",
"- dlt's DTC cohort channel: https://dlthub-community.slack.com/archives/C06GAEX2VNX"
],
"metadata": {
"id": "PoTJu4kbGG0z"
}
}
]
}

BIN
cohorts/2024/workshops/dlt_resources/incremental_loading.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 26 KiB

14645
cohorts/2024/workshops/dlt_resources/workshop.ipynb
View File

File diff suppressed because one or more lines are too long

176
cohorts/2024/workshops/rising-wave.md
View File

@ -1,176 +0,0 @@
<p align="center">
<picture>
<source srcset="https://github.com/risingwavelabs/risingwave/blob/main/.github/RisingWave-logo-dark.svg" width="500px" media="(prefers-color-scheme: dark)">
<img src="https://github.com/risingwavelabs/risingwave/blob/main/.github/RisingWave-logo-light.svg" width="500px">
</picture>
</p>
</div>
<p align="center">
<a
href="https://docs.risingwave.com/"
target="_blank"
><b>Documentation</b></a>&nbsp;&nbsp;&nbsp;📑&nbsp;&nbsp;&nbsp;
<a
href="https://tutorials.risingwave.com/"
target="_blank"
><b>Hands-on Tutorials</b></a>&nbsp;&nbsp;&nbsp;🎯&nbsp;&nbsp;&nbsp;
<a
href="https://cloud.risingwave.com/"
target="_blank"
><b>RisingWave Cloud</b></a>&nbsp;&nbsp;&nbsp;🚀&nbsp;&nbsp;&nbsp;
<a
href="https://risingwave.com/slack"
target="_blank"
>
<b>Get Instant Help</b>
</a>
</p>
<div align="center">
<a
href="https://risingwave.com/slack"
target="_blank"
>
<img alt="Slack" src="https://badgen.net/badge/Slack/Join%20RisingWave/0abd59?icon=slack" />
</a>
<a
href="https://twitter.com/risingwavelabs"
target="_blank"
>
<img alt="X" src="https://img.shields.io/twitter/follow/risingwavelabs" />
</a>
<a
href="https://www.youtube.com/@risingwave-labs"
target="_blank"
>
<img alt="YouTube" src="https://img.shields.io/youtube/channel/views/UCsHwdyBRxBpmkA5RRd0YNEA" />
</a>
</div>
## Stream processing with RisingWave
In this hands-on workshop, we’ll learn how to process real-time streaming data using SQL in RisingWave. The system we’ll use is [RisingWave](https://github.com/risingwavelabs/risingwave), an open-source SQL database for processing and managing streaming data. You may not feel unfamiliar with RisingWave’s user experience, as it’s fully wire compatible with PostgreSQL.
![RisingWave](https://raw.githubusercontent.com/risingwavelabs/risingwave-docs/main/docs/images/new_archi_grey.png)
We’ll cover the following topics in this Workshop:
- Why Stream Processing?
- Stateless computation (Filters, Projections)
- Stateful Computation (Aggregations, Joins)
- Data Ingestion and Delivery
RisingWave in 10 Minutes:
https://tutorials.risingwave.com/docs/intro
Workshop video:
<a href="https://youtube.com/live/L2BHFnZ6XjE">
<img src="https://markdown-videos-api.jorgenkh.no/youtube/L2BHFnZ6XjE" />
</a>
[Project Repository](https://github.com/risingwavelabs/risingwave-data-talks-workshop-2024-03-04)
## Homework
**Please setup the environment in [Getting Started](https://github.com/risingwavelabs/risingwave-data-talks-workshop-2024-03-04?tab=readme-ov-file#getting-started) and for the [Homework](https://github.com/risingwavelabs/risingwave-data-talks-workshop-2024-03-04/blob/main/homework.md#setting-up) first.**
## Question 0
_This question is just a warm-up to introduce dynamic filter, please attempt it before viewing its solution._
What are the dropoff taxi zones at the latest dropoff times?
For this part, we will use the [dynamic filter pattern](https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/).
<details>
<summary>Solution</summary>
```sql
CREATE MATERIALIZED VIEW latest_dropoff_time AS
WITH t AS (
SELECT MAX(tpep_dropoff_datetime) AS latest_dropoff_time
FROM trip_data
)
SELECT taxi_zone.Zone as taxi_zone, latest_dropoff_time
FROM t,
trip_data
JOIN taxi_zone
ON trip_data.DOLocationID = taxi_zone.location_id
WHERE trip_data.tpep_dropoff_datetime = t.latest_dropoff_time;
-- taxi_zone | latest_dropoff_time
-- ----------------+---------------------
-- Midtown Center | 2022-01-03 17:24:54
-- (1 row)
```
</details>
### Question 1
Create a materialized view to compute the average, min and max trip time between each taxi zone.
From this MV, find the pair of taxi zones with the highest average trip time.
You may need to use the [dynamic filter pattern](https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/) for this.
Bonus (no marks): Create an MV which can identify anomalies in the data. For example, if the average trip time between two zones is 1 minute,
but the max trip time is 10 minutes and 20 minutes respectively.
Options:
1. Yorkville East, Steinway
2. Murray Hill, Midwood
3. East Flatbush/Farragut, East Harlem North
4. Midtown Center, University Heights/Morris Heights
### Question 2
Recreate the MV(s) in question 1, to also find the number of trips for the pair of taxi zones with the highest average trip time.
Options:
1. 5
2. 3
3. 10
4. 1
### Question 3
From the latest pickup time to 17 hours before, what are the top 3 busiest zones in terms of number of pickups?
For example if the latest pickup time is 2020-01-01 12:00:00,
then the query should return the top 3 busiest zones from 2020-01-01 11:00:00 to 2020-01-01 12:00:00.
HINT: You can use [dynamic filter pattern](https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/)
to create a filter condition based on the latest pickup time.
NOTE: For this question `17 hours` was picked to ensure we have enough data to work with.
Options:
1. Clinton East, Upper East Side North, Penn Station
2. LaGuardia Airport, Lincoln Square East, JFK Airport
3. Midtown Center, Upper East Side South, Upper East Side North
4. LaGuardia Airport, Midtown Center, Upper East Side North
## Submitting the solutions
- Form for submitting: https://courses.datatalks.club/de-zoomcamp-2024/homework/workshop2
- Deadline: 11 March (Monday), 23:00 CET
## Rewards 🥳
Everyone who completes the homework will get a pen and a sticker, and 5 lucky winners will receive a Tshirt and other secret surprises!
We encourage you to share your achievements with this workshop on your socials and look forward to your submissions 😁
- Follow us on **LinkedIn**: https://www.linkedin.com/company/risingwave
- Follow us on **GitHub**: https://github.com/risingwavelabs/risingwave
- Join us on **Slack**: https://risingwave-labs.com/slack
See you around!
## Solution

BIN
images/architecture/arch_2.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 163 KiB

BIN
images/architecture/arch_v3_workshops.jpg
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 309 KiB

BIN
images/architecture/photo1700757552.jpeg
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 66 KiB

BIN
images/rising-wave.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 30 KiB

0
01-docker-terraform/1_terraform_gcp/1_terraform_overview.md → week_1_basics_n_setup/1_terraform_gcp/1_terraform_overview.md
View File

0
01-docker-terraform/1_terraform_gcp/2_gcp_overview.md → week_1_basics_n_setup/1_terraform_gcp/2_gcp_overview.md
View File

0
01-docker-terraform/1_terraform_gcp/README.md → week_1_basics_n_setup/1_terraform_gcp/README.md
View File

0
01-docker-terraform/1_terraform_gcp/terraform/README.md → week_1_basics_n_setup/1_terraform_gcp/terraform/README.md
View File

0
01-docker-terraform/1_terraform_gcp/terraform/terraform_basic/main.tf → week_1_basics_n_setup/1_terraform_gcp/terraform/terraform_basic/main.tf
View File

0
01-docker-terraform/1_terraform_gcp/terraform/terraform_with_variables/main.tf → week_1_basics_n_setup/1_terraform_gcp/terraform/terraform_with_variables/main.tf
View File

0
01-docker-terraform/1_terraform_gcp/terraform/terraform_with_variables/variables.tf → week_1_basics_n_setup/1_terraform_gcp/terraform/terraform_with_variables/variables.tf
View File

0
01-docker-terraform/1_terraform_gcp/windows.md → week_1_basics_n_setup/1_terraform_gcp/windows.md
View File

0
01-docker-terraform/2_docker_sql/.gitignore → week_1_basics_n_setup/2_docker_sql/.gitignore vendored
View File

0
01-docker-terraform/2_docker_sql/Dockerfile → week_1_basics_n_setup/2_docker_sql/Dockerfile
View File

4
01-docker-terraform/2_docker_sql/README.md → week_1_basics_n_setup/2_docker_sql/README.md
View File

@ -113,10 +113,6 @@ $ aws s3 ls s3://nyc-tlc
PRE trip data/
```
You can refer the `data-loading-parquet.ipynb` and `data-loading-parquet.py` for code to handle both csv and paraquet files. (The lookup zones table which is needed later in this course is a csv file)
> Note: You will need to install the `pyarrow` library. (add it to your Dockerfile)
### pgAdmin
Running pgAdmin

0
01-docker-terraform/2_docker_sql/docker-compose.yaml → week_1_basics_n_setup/2_docker_sql/docker-compose.yaml
View File

0
01-docker-terraform/2_docker_sql/ingest_data.py → week_1_basics_n_setup/2_docker_sql/ingest_data.py
View File

0
01-docker-terraform/2_docker_sql/pg-test-connection.ipynb → week_1_basics_n_setup/2_docker_sql/pg-test-connection.ipynb
View File

0
01-docker-terraform/2_docker_sql/pipeline.py → week_1_basics_n_setup/2_docker_sql/pipeline.py
View File

0
01-docker-terraform/2_docker_sql/upload-data.ipynb → week_1_basics_n_setup/2_docker_sql/upload-data.ipynb
View File

127
week_1_basics_n_setup/README.md Normal file
View File

@ -0,0 +1,127 @@
### Introduction
* [Video](https://www.youtube.com/watch?v=-zpVha7bw5A)
* [Slides](https://www.slideshare.net/AlexeyGrigorev/data-engineering-zoomcamp-introduction)
* Overview of [Architecture](https://github.com/DataTalksClub/data-engineering-zoomcamp#overview), [Technologies](https://github.com/DataTalksClub/data-engineering-zoomcamp#technologies) & [Pre-Requisites](https://github.com/DataTalksClub/data-engineering-zoomcamp#prerequisites)
We suggest watching videos in the same order as in this document.
The last video (setting up the environment) is optional, but you can check it earlier
if you have troubles setting up the environment and following along with the videos.
### Docker + Postgres
[Code](2_docker_sql)
* [Introduction to Docker](https://www.youtube.com/watch?v=EYNwNlOrpr0&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* Why do we need Docker
* Creating a simple "data pipeline" in Docker
* [Ingesting NY Taxi Data to Postgres](https://www.youtube.com/watch?v=2JM-ziJt0WI&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* Running Postgres locally with Docker
* Using `pgcli` for connecting to the database
* Exploring the NY Taxi dataset
* Ingesting the data into the database
* **Note** if you have problems with `pgcli`, check [this video](https://www.youtube.com/watch?v=3IkfkTwqHx4&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
for an alternative way to connect to your database
* [Connecting pgAdmin and Postgres](https://www.youtube.com/watch?v=hCAIVe9N0ow&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* The pgAdmin tool
* Docker networks
* [Putting the ingestion script into Docker](https://www.youtube.com/watch?v=B1WwATwf-vY&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* Converting the Jupyter notebook to a Python script
* Parametrizing the script with argparse
* Dockerizing the ingestion script
* [Running Postgres and pgAdmin with Docker-Compose](https://www.youtube.com/watch?v=hKI6PkPhpa0&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* Why do we need Docker-compose
* Docker-compose YAML file
* Running multiple containers with `docker-compose up`
* [SQL refresher](https://www.youtube.com/watch?v=QEcps_iskgg&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* Adding the Zones table
* Inner joins
* Basic data quality checks
* Left, Right and Outer joins
* Group by
* Optional: If you have some problems with docker networking, check [Port Mapping and Networks in Docker](https://www.youtube.com/watch?v=tOr4hTsHOzU&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* Docker networks
* Port forwarding to the host environment
* Communicating between containers in the network
* `.dockerignore` file
* Optional: If you are willing to do the steps from "Ingesting NY Taxi Data to Postgres" till "Running Postgres and pgAdmin with Docker-Compose" with Windows Subsystem Linux please check [Docker Module Walk-Through on WSL](https://www.youtube.com/watch?v=Mv4zFm2AwzQ)
### GCP + Terraform
[Code](1_terraform_gcp)
* Introduction to GCP (Google Cloud Platform)
* [Video](https://www.youtube.com/watch?v=18jIzE41fJ4&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* Introduction to Terraform Concepts - Overview
* [Video](https://youtu.be/s2bOYDCKl_M)
* [Companion Notes](1_terraform_gcp)
* Terraform Basice - Simple one file Terraform Deployment
* [Video](https://youtu.be/Y2ux7gq3Z0o)
* [Companion Notes](1_terraform_gcp)
* Terraform Continued - Terraform Deployment with a Variables File
* [Video](https://youtu.be/PBi0hHjLftk)
* [Companion Notes](1_terraform_gcp)
* Configuring terraform and GCP SDK on Windows
* [Instructions](1_terraform_gcp/windows.md)
### Environment setup
For the course you'll need:
* Python 3 (e.g. installed with Anaconda)
* Google Cloud SDK
* Docker with docker-compose
* Terraform
If you have problems setting up the env, you can check this video:
* [Setting up the environment on cloud VM](https://www.youtube.com/watch?v=ae-CV2KfoN0&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* Generating SSH keys
* Creating a virtual machine on GCP
* Connecting to the VM with SSH
* Installing Anaconda
* Installing Docker
* Creating SSH `config` file
* Accessing the remote machine with VS Code and SSH remote
* Installing docker-compose
* Installing pgcli
* Port-forwarding with VS code: connecting to pgAdmin and Jupyter from the local computer
* Installing Terraform
* Using `sftp` for putting the credentials to the remote machine
* Shutting down and removing the instance
### Homework
* [Homework](../cohorts/2023/week_1_docker_sql/homework.md)
* [Homework-PartB](../cohorts/2023/week_1_terraform/homework.md)
## Community notes
Did you take notes? You can share them here
* [Notes from Alvaro Navas](https://github.com/ziritrion/dataeng-zoomcamp/blob/main/notes/1_intro.md)
* [Notes from Abd](https://itnadigital.notion.site/Week-1-Introduction-f18de7e69eb4453594175d0b1334b2f4)
* [Notes from Aaron](https://github.com/ABZ-Aaron/DataEngineerZoomCamp/blob/master/week_1_basics_n_setup/README.md)
* [Notes from Faisal](https://github.com/FaisalMohd/data-engineering-zoomcamp/blob/main/week_1_basics_n_setup/Notes/DE%20Zoomcamp%20Week-1.pdf)
* [Michael Harty's Notes](https://github.com/mharty3/data_engineering_zoomcamp_2022/tree/main/week01)
* [Blog post from Isaac Kargar](https://kargarisaac.github.io/blog/data%20engineering/jupyter/2022/01/18/data-engineering-w1.html)
* [Handwritten Notes By Mahmoud Zaher](https://github.com/zaherweb/DataEngineering/blob/master/week%201.pdf)
* [Notes from Candace Williams](https://teacherc.github.io/data-engineering/2023/01/18/zoomcamp1.html)
* [Notes from Marcos Torregrosa](https://www.n4gash.com/2023/data-engineering-zoomcamp-semana-1/)
* [Notes from Vincenzo Galante](https://binchentso.notion.site/Data-Talks-Club-Data-Engineering-Zoomcamp-8699af8e7ff94ec49e6f9bdec8eb69fd)
* [Notes from Victor Padilha](https://github.com/padilha/de-zoomcamp/tree/master/week1)
* [Notes from froukje](https://github.com/froukje/de-zoomcamp/blob/main/week_1_basics_n_setup/notes/notes_week_01.md)
* [Notes from adamiaonr](https://github.com/adamiaonr/data-engineering-zoomcamp/blob/main/week_1_basics_n_setup/2_docker_sql/NOTES.md)
* [Notes from Xia He-Bleinagel](https://xiahe-bleinagel.com/2023/01/week-1-data-engineering-zoomcamp-notes/)
* [Notes from Balaji](https://github.com/Balajirvp/DE-Zoomcamp/blob/main/Week%201/Detailed%20Week%201%20Notes.ipynb)
* [Notes from Erik](https://twitter.com/ehub96/status/1621351266281730049)
* [Notes by Alain Boisvert](https://github.com/boisalai/de-zoomcamp-2023/blob/main/week1.md)
* Notes on [Docker, Docker Compose, and setting up a proper Python environment](https://medium.com/@verazabeida/zoomcamp-2023-week-1-f4f94cb360ae), by Vera
* [Setting up the development environment on Google Virtual Machine](https://itsadityagupta.hashnode.dev/setting-up-the-development-environment-on-google-virtual-machine), blog post by Aditya Gupta
* Add your notes here

56
week_2_workflow_orchestration/README.md Normal file
View File

@ -0,0 +1,56 @@
## Week 2: Workflow Orchestration
> If you're looking for Airflow videos from the 2022 edition,
> check the [2022 cohort folder](../cohorts/2022/week_2_data_ingestion/).
Python code from videos is linked [below](#code-repository).
Also, if you find the commands too small to view in Kalise's videos, here's the [transcript with code for the second Prefect video](https://github.com/discdiver/prefect-zoomcamp/tree/main/flows/01_start) and the [fifth Prefect video](https://github.com/discdiver/prefect-zoomcamp/tree/main/flows/03_deployments).
### Data Lake (GCS)
* What is a Data Lake
* ELT vs. ETL
* Alternatives to components (S3/HDFS, Redshift, Snowflake etc.)
* [Video](https://www.youtube.com/watch?v=W3Zm6rjOq70&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb)
* [Slides](https://docs.google.com/presentation/d/1RkH-YhBz2apIjYZAxUz2Uks4Pt51-fVWVN9CcH9ckyY/edit?usp=sharing)
### Workflow Orchestration
Mage videos coming soon
### Code repository
[Code from videos](https://github.com/discdiver/prefect-zoomcamp) (with a few minor enhancements)
### Homework
Homework can be found [here](../cohorts/2023/week_2_workflow_orchestration/homework.md).
## Community notes
Did you take notes? You can share them here.
* Add your notes here (above this line)
### 2023 notes
* [Blog by Marcos Torregrosa (Prefect)](https://www.n4gash.com/2023/data-engineering-zoomcamp-semana-2/)
* [Notes from Victor Padilha](https://github.com/padilha/de-zoomcamp/tree/master/week2)
* [Notes by Alain Boisvert](https://github.com/boisalai/de-zoomcamp-2023/blob/main/week2.md)
* [Notes by Candace Williams](https://github.com/teacherc/de_zoomcamp_candace2023/blob/main/week_2/week2_notes.md)
* [Notes from Xia He-Bleinagel](https://xiahe-bleinagel.com/2023/02/week-2-data-engineering-zoomcamp-notes-prefect/)
* [Notes from froukje](https://github.com/froukje/de-zoomcamp/blob/main/week_2_workflow_orchestration/notes/notes_week_02.md)
* [Notes from Balaji](https://github.com/Balajirvp/DE-Zoomcamp/blob/main/Week%202/Detailed%20Week%202%20Notes.ipynb)
### 2022 notes
Most of these notes are about Airflow, but you might find them useful.
* [Notes from Alvaro Navas](https://github.com/ziritrion/dataeng-zoomcamp/blob/main/notes/2_data_ingestion.md)
* [Notes from Aaron Wright](https://github.com/ABZ-Aaron/DataEngineerZoomCamp/blob/master/week_2_data_ingestion/README.md)
* [Notes from Abd](https://itnadigital.notion.site/Week-2-Data-Ingestion-ec2d0d36c0664bc4b8be6a554b2765fd)
* [Blog post by Isaac Kargar](https://kargarisaac.github.io/blog/data%20engineering/jupyter/2022/01/25/data-engineering-w2.html)
* [Blog, notes, walkthroughs by Sandy Behrens](https://learningdataengineering540969211.wordpress.com/2022/01/30/week-2-de-zoomcamp-2-3-2-ingesting-data-to-gcp-with-airflow/)
* [Notes from Vincenzo Galante](https://binchentso.notion.site/Data-Talks-Club-Data-Engineering-Zoomcamp-8699af8e7ff94ec49e6f9bdec8eb69fd)
* More on [Pandas vs SQL, Prefect capabilities, and testing your data](https://medium.com/@verazabeida/zoomcamp-2023-week-3-7f27bb8c483f), by Vera

61
week_3_data_warehouse/README.md Normal file
View File

@ -0,0 +1,61 @@
## Data Warehouse and BigQuery
- [Slides](https://docs.google.com/presentation/d/1a3ZoBAXFk8-EhUsd7rAZd-5p_HpltkzSeujjRGB2TAI/edit?usp=sharing)
- [Big Query basic SQL](big_query.sql)
### Data Warehouse
- [Data Warehouse and BigQuery](https://youtu.be/jrHljAoD6nM)
### Partitoning and clustering
- [Partioning and Clustering](https://youtu.be/jrHljAoD6nM?t=726)
- [Partioning vs Clustering](https://youtu.be/-CqXf7vhhDs)
### Best practices
- [BigQuery Best Practices](https://youtu.be/k81mLJVX08w)
### Internals of BigQuery
- [Internals of Big Query](https://youtu.be/eduHi1inM4s)
### Advanced
#### ML
[BigQuery Machine Learning](https://youtu.be/B-WtpB0PuG4)
[SQL for ML in BigQuery](big_query_ml.sql)
**Important links**
- [BigQuery ML Tutorials](https://cloud.google.com/bigquery-ml/docs/tutorials)
- [BigQuery ML Reference Parameter](https://cloud.google.com/bigquery-ml/docs/analytics-reference-patterns)
- [Hyper Parameter tuning](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create-glm)
- [Feature preprocessing](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-preprocess-overview)
##### Deploying ML model
- [BigQuery Machine Learning Deployment](https://youtu.be/BjARzEWaznU)
- [Steps to extract and deploy model with docker](extract_model.md)
### Homework
* [Homework](../cohorts/2023/week_3_data_warehouse/homework.md)
## Community notes
Did you take notes? You can share them here.
* [Notes by Alvaro Navas](https://github.com/ziritrion/dataeng-zoomcamp/blob/main/notes/3_data_warehouse.md)
* [Isaac Kargar's blog post](https://kargarisaac.github.io/blog/data%20engineering/jupyter/2022/01/30/data-engineering-w3.html)
* [Marcos Torregrosa's blog post](https://www.n4gash.com/2023/data-engineering-zoomcamp-semana-3/)
* [Notes by Victor Padilha](https://github.com/padilha/de-zoomcamp/tree/master/week3)
* [Notes from Xia He-Bleinagel](https://xiahe-bleinagel.com/2023/02/week-3-data-engineering-zoomcamp-notes-data-warehouse-and-bigquery/)
* [Bigger picture summary on Data Lakes, Data Warehouses, and tooling](https://medium.com/@verazabeida/zoomcamp-week-4-b8bde661bf98), by Vera
* [Notes by froukje](https://github.com/froukje/de-zoomcamp/blob/main/week_3_data_warehouse/notes/notes_week_03.md)
* [Notes by Alain Boisvert](https://github.com/boisalai/de-zoomcamp-2023/blob/main/week3.md)
* [Notes from Vincenzo Galante](https://binchentso.notion.site/Data-Talks-Club-Data-Engineering-Zoomcamp-8699af8e7ff94ec49e6f9bdec8eb69fd)
* Add your notes here (above this line)

0
03-data-warehouse/big_query.sql → week_3_data_warehouse/big_query.sql
View File

0
03-data-warehouse/big_query_hw.sql → week_3_data_warehouse/big_query_hw.sql
View File

0
03-data-warehouse/big_query_ml.sql → week_3_data_warehouse/big_query_ml.sql
View File

0
03-data-warehouse/extract_model.md → week_3_data_warehouse/extract_model.md
View File

0
03-data-warehouse/extras/README.md → week_3_data_warehouse/extras/README.md
View File

0
03-data-warehouse/extras/web_to_gcs.py → week_3_data_warehouse/extras/web_to_gcs.py
View File

136
week_4_analytics_engineering/README.md Normal file
View File

@ -0,0 +1,136 @@
# Week 4: Analytics Engineering
Goal: Transforming the data loaded in DWH to Analytical Views developing a [dbt project](taxi_rides_ny/README.md).
[Slides](https://docs.google.com/presentation/d/1xSll_jv0T8JF4rYZvLHfkJXYqUjPtThA/edit?usp=sharing&ouid=114544032874539580154&rtpof=true&sd=true)
## Prerequisites
We will build a project using dbt and a running data warehouse.
By this stage of the course you should have already:
- A running warehouse (BigQuery or postgres)
- A set of running pipelines ingesting the project dataset (week 3 completed): [Datasets list](https://github.com/DataTalksClub/nyc-tlc-data/)
* Yellow taxi data - Years 2019 and 2020
* Green taxi data - Years 2019 and 2020
* fhv data - Year 2019.
_Note:_
* _A quick hack has been shared to load that data quicker, check instructions in [week3/extras](https://github.com/DataTalksClub/data-engineering-zoomcamp/tree/main/week_3_data_warehouse/extras)_
* _If you recieve an error stating "Permission denied while globbing file pattern." when attemting to run fact_trips.sql this video may be helpful in resolving the issue_
:movie_camera: [Video](https://www.youtube.com/watch?v=kL3ZVNL9Y4A)
### Setting up dbt for using BigQuery (Alternative A - preferred)
You will need to create a dbt cloud account using [this link](https://www.getdbt.com/signup/) and connect to your warehouse [following these instructions](https://docs.getdbt.com/docs/dbt-cloud/cloud-configuring-dbt-cloud/cloud-setting-up-bigquery-oauth). More detailed instructions in [dbt_cloud_setup.md](dbt_cloud_setup.md)
_Optional_: If you feel more comfortable developing locally you could use a local installation of dbt as well. You can follow the [official dbt documentation](https://docs.getdbt.com/dbt-cli/installation) or follow the [dbt with BigQuery on Docker](docker_setup/README.md) guide to setup dbt locally on docker. You will need to install the latest version (1.0) with the BigQuery adapter (dbt-bigquery).
### Setting up dbt for using Postgres locally (Alternative B)
As an alternative to the cloud, that require to have a cloud database, you will be able to run the project installing dbt locally.
You can follow the [official dbt documentation](https://docs.getdbt.com/dbt-cli/installation) or use a docker image from oficial [dbt repo](https://github.com/dbt-labs/dbt/). You will need to install the latest version (1.0) with the postgres adapter (dbt-postgres).
After local installation you will have to set up the connection to PG in the `profiles.yml`, you can find the templates [here](https://docs.getdbt.com/reference/warehouse-profiles/postgres-profile)
## Content
### Introduction to analytics engineering
* What is analytics engineering?
* ETL vs ELT
* Data modeling concepts (fact and dim tables)
:movie_camera: [Video](https://www.youtube.com/watch?v=uF76d5EmdtU&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=32)
### What is dbt?
* Intro to dbt
:movie_camera: [Video](https://www.youtube.com/watch?v=4eCouvVOJUw&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=33)
### Starting a dbt project
#### Alternative a: Using BigQuery + dbt cloud
* Starting a new project with dbt init (dbt cloud and core)
* dbt cloud setup
* project.yml
:movie_camera: [Video](https://www.youtube.com/watch?v=iMxh6s_wL4Q&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=34)
#### Alternative b: Using Postgres + dbt core (locally)
* Starting a new project with dbt init (dbt cloud and core)
* dbt core local setup
* profiles.yml
* project.yml
:movie_camera: [Video](https://www.youtube.com/watch?v=1HmL63e-vRs&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=35)
### Development of dbt models
* Anatomy of a dbt model: written code vs compiled Sources
* Materialisations: table, view, incremental, ephemeral
* Seeds, sources and ref
* Jinja and Macros
* Packages
* Variables
:movie_camera: [Video](https://www.youtube.com/watch?v=UVI30Vxzd6c&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=36)
_Note: This video is shown entirely on dbt cloud IDE but the same steps can be followed locally on the IDE of your choice_
### Testing and documenting dbt models
* Tests
* Documentation
:movie_camera: [Video](https://www.youtube.com/watch?v=UishFmq1hLM&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=37)
_Note: This video is shown entirely on dbt cloud IDE but the same steps can be followed locally on the IDE of your choice_
### Deploying a dbt project
#### Alternative a: Using BigQuery + dbt cloud
* Deployment: development environment vs production
* dbt cloud: scheduler, sources and hosted documentation
:movie_camera: [Video](https://www.youtube.com/watch?v=rjf6yZNGX8I&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=38)
#### Alternative b: Using Postgres + dbt core (locally)
* Deployment: development environment vs production
* dbt cloud: scheduler, sources and hosted documentation
:movie_camera: [Video](https://www.youtube.com/watch?v=Cs9Od1pcrzM&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=39)
### Visualising the transformed data
* Google data studio
* [Metabase (local installation)](https://www.metabase.com/)
:movie_camera: [Google data studio Video](https://www.youtube.com/watch?v=39nLTs74A3E&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=42)
:movie_camera: [Metabase Video](https://www.youtube.com/watch?v=BnLkrA7a6gM&list=PL3MmuxUbc_hJed7dXYoJw8DoCuVHhGEQb&index=43)
### Advanced knowledge:
* [Make a model Incremental](https://docs.getdbt.com/docs/building-a-dbt-project/building-models/configuring-incremental-models)
* [Use of tags](https://docs.getdbt.com/reference/resource-configs/tags)
* [Hooks](https://docs.getdbt.com/docs/building-a-dbt-project/hooks-operations)
* [Analysis](https://docs.getdbt.com/docs/building-a-dbt-project/analyses)
* [Snapshots](https://docs.getdbt.com/docs/building-a-dbt-project/snapshots)
* [Exposure](https://docs.getdbt.com/docs/building-a-dbt-project/exposures)
* [Metrics](https://docs.getdbt.com/docs/building-a-dbt-project/metrics)
## Workshop: Maximizing Confidence in Your Data Model Changes with dbt and PipeRider
To learn how to use PipeRider together with dbt for detecting changes in model and data, sign up for a workshop [here](https://www.eventbrite.com/e/maximizing-confidence-in-your-data-model-changes-with-dbt-and-piperider-tickets-535584366257)
[More details](../cohorts/2023/workshops/piperider.md)
## Community notes
Did you take notes? You can share them here.
* [Notes by Alvaro Navas](https://github.com/ziritrion/dataeng-zoomcamp/blob/main/notes/4_analytics.md)
* [Sandy's DE learning blog](https://learningdataengineering540969211.wordpress.com/2022/02/17/week-4-setting-up-dbt-cloud-with-bigquery/)
* [Notes by Victor Padilha](https://github.com/padilha/de-zoomcamp/tree/master/week4)
* [Marcos Torregrosa's blog (spanish)](https://www.n4gash.com/2023/data-engineering-zoomcamp-semana-4/)
* [Notes by froukje](https://github.com/froukje/de-zoomcamp/blob/main/week_4_analytics_engineering/notes/notes_week_04.md)
* [Notes by Alain Boisvert](https://github.com/boisalai/de-zoomcamp-2023/blob/main/week4.md)
* [Setting up Prefect with dbt by Vera](https://medium.com/@verazabeida/zoomcamp-week-5-5b6a9d53a3a0)
* [Blog by Xia He-Bleinagel](https://xiahe-bleinagel.com/2023/02/week-4-data-engineering-zoomcamp-notes-analytics-engineering-and-dbt/)
* [Setting up DBT with BigQuery by Tofag](https://medium.com/@fagbuyit/setting-up-your-dbt-cloud-dej-9-d18e5b7c96ba)
* [Blog post by Dewi Oktaviani](https://medium.com/@oktavianidewi/de-zoomcamp-2023-learning-week-4-analytics-engineering-with-dbt-53f781803d3e)
* [Notes from Vincenzo Galante](https://binchentso.notion.site/Data-Talks-Club-Data-Engineering-Zoomcamp-8699af8e7ff94ec49e6f9bdec8eb69fd)
* [Notes from Balaji](https://github.com/Balajirvp/DE-Zoomcamp/blob/main/Week%204/Data%20Engineering%20Zoomcamp%20Week%204.ipynb)
* Add your notes here (above this line)
## Useful links
- [Visualizing data with Metabase course](https://www.metabase.com/learn/visualization/)

0
04-analytics-engineering/dbt_cloud_setup.md → week_4_analytics_engineering/dbt_cloud_setup.md
View File

0
04-analytics-engineering/docker_setup/Dockerfile → week_4_analytics_engineering/docker_setup/Dockerfile
View File

0
04-analytics-engineering/docker_setup/README.md → week_4_analytics_engineering/docker_setup/README.md
View File

0
04-analytics-engineering/docker_setup/docker-compose.yaml → week_4_analytics_engineering/docker_setup/docker-compose.yaml
View File

5
week_4_analytics_engineering/taxi_rides_ny/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
target/
dbt_modules/
logs/
dbt_packages/

2
04-analytics-engineering/taxi_rides_ny/README.md → week_4_analytics_engineering/taxi_rides_ny/README.md
View File

@ -35,4 +35,4 @@ _Alternative: use `$ dbt build` to execute with one command the 3 steps above to
- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
- Join the [chat](http://slack.getdbt.com/) on Slack for live discussions and support
- Find [dbt events](https://events.getdbt.com) near you
- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices
- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

0
04-analytics-engineering/taxi_rides_ny/.gitkeep → week_4_analytics_engineering/taxi_rides_ny/analysis/.gitkeep
View File

0
04-analytics-engineering/taxi_rides_ny/analyses/.gitkeep → week_4_analytics_engineering/taxi_rides_ny/data/.gitkeep
View File

3
04-analytics-engineering/taxi_rides_ny/seeds/seeds_properties.yml → week_4_analytics_engineering/taxi_rides_ny/data/seeds_properties.yml
View File

@ -6,4 +6,5 @@ seeds:
Taxi Zones roughly based on NYC Department of City Planning's Neighborhood
Tabulation Areas (NTAs) and are meant to approximate neighborhoods, so you can see which
neighborhood a passenger was picked up in, and which neighborhood they were dropped off in.
Includes associated service_zone (EWR, Boro Zone, Yellow Zone)
Includes associated service_zone (EWR, Boro Zone, Yellow Zone)

532
04-analytics-engineering/taxi_rides_ny/seeds/taxi_zone_lookup.csv → week_4_analytics_engineering/taxi_rides_ny/data/taxi_zone_lookup.csv
View File

@ -1,266 +1,266 @@
"locationid","borough","zone","service_zone"
1,"EWR","Newark Airport","EWR"
2,"Queens","Jamaica Bay","Boro Zone"
3,"Bronx","Allerton/Pelham Gardens","Boro Zone"
4,"Manhattan","Alphabet City","Yellow Zone"
5,"Staten Island","Arden Heights","Boro Zone"
6,"Staten Island","Arrochar/Fort Wadsworth","Boro Zone"
7,"Queens","Astoria","Boro Zone"
8,"Queens","Astoria Park","Boro Zone"
9,"Queens","Auburndale","Boro Zone"
10,"Queens","Baisley Park","Boro Zone"
11,"Brooklyn","Bath Beach","Boro Zone"
12,"Manhattan","Battery Park","Yellow Zone"
13,"Manhattan","Battery Park City","Yellow Zone"
14,"Brooklyn","Bay Ridge","Boro Zone"
15,"Queens","Bay Terrace/Fort Totten","Boro Zone"
16,"Queens","Bayside","Boro Zone"
17,"Brooklyn","Bedford","Boro Zone"
18,"Bronx","Bedford Park","Boro Zone"
19,"Queens","Bellerose","Boro Zone"
20,"Bronx","Belmont","Boro Zone"
21,"Brooklyn","Bensonhurst East","Boro Zone"
22,"Brooklyn","Bensonhurst West","Boro Zone"
23,"Staten Island","Bloomfield/Emerson Hill","Boro Zone"
24,"Manhattan","Bloomingdale","Yellow Zone"
25,"Brooklyn","Boerum Hill","Boro Zone"
26,"Brooklyn","Borough Park","Boro Zone"
27,"Queens","Breezy Point/Fort Tilden/Riis Beach","Boro Zone"
28,"Queens","Briarwood/Jamaica Hills","Boro Zone"
29,"Brooklyn","Brighton Beach","Boro Zone"
30,"Queens","Broad Channel","Boro Zone"
31,"Bronx","Bronx Park","Boro Zone"
32,"Bronx","Bronxdale","Boro Zone"
33,"Brooklyn","Brooklyn Heights","Boro Zone"
34,"Brooklyn","Brooklyn Navy Yard","Boro Zone"
35,"Brooklyn","Brownsville","Boro Zone"
36,"Brooklyn","Bushwick North","Boro Zone"
37,"Brooklyn","Bushwick South","Boro Zone"
38,"Queens","Cambria Heights","Boro Zone"
39,"Brooklyn","Canarsie","Boro Zone"
40,"Brooklyn","Carroll Gardens","Boro Zone"
41,"Manhattan","Central Harlem","Boro Zone"
42,"Manhattan","Central Harlem North","Boro Zone"
43,"Manhattan","Central Park","Yellow Zone"
44,"Staten Island","Charleston/Tottenville","Boro Zone"
45,"Manhattan","Chinatown","Yellow Zone"
46,"Bronx","City Island","Boro Zone"
47,"Bronx","Claremont/Bathgate","Boro Zone"
48,"Manhattan","Clinton East","Yellow Zone"
49,"Brooklyn","Clinton Hill","Boro Zone"
50,"Manhattan","Clinton West","Yellow Zone"
51,"Bronx","Co-Op City","Boro Zone"
52,"Brooklyn","Cobble Hill","Boro Zone"
53,"Queens","College Point","Boro Zone"
54,"Brooklyn","Columbia Street","Boro Zone"
55,"Brooklyn","Coney Island","Boro Zone"
56,"Queens","Corona","Boro Zone"
57,"Queens","Corona","Boro Zone"
58,"Bronx","Country Club","Boro Zone"
59,"Bronx","Crotona Park","Boro Zone"
60,"Bronx","Crotona Park East","Boro Zone"
61,"Brooklyn","Crown Heights North","Boro Zone"
62,"Brooklyn","Crown Heights South","Boro Zone"
63,"Brooklyn","Cypress Hills","Boro Zone"
64,"Queens","Douglaston","Boro Zone"
65,"Brooklyn","Downtown Brooklyn/MetroTech","Boro Zone"
66,"Brooklyn","DUMBO/Vinegar Hill","Boro Zone"
67,"Brooklyn","Dyker Heights","Boro Zone"
68,"Manhattan","East Chelsea","Yellow Zone"
69,"Bronx","East Concourse/Concourse Village","Boro Zone"
70,"Queens","East Elmhurst","Boro Zone"
71,"Brooklyn","East Flatbush/Farragut","Boro Zone"
72,"Brooklyn","East Flatbush/Remsen Village","Boro Zone"
73,"Queens","East Flushing","Boro Zone"
74,"Manhattan","East Harlem North","Boro Zone"
75,"Manhattan","East Harlem South","Boro Zone"
76,"Brooklyn","East New York","Boro Zone"
77,"Brooklyn","East New York/Pennsylvania Avenue","Boro Zone"
78,"Bronx","East Tremont","Boro Zone"
79,"Manhattan","East Village","Yellow Zone"
80,"Brooklyn","East Williamsburg","Boro Zone"
81,"Bronx","Eastchester","Boro Zone"
82,"Queens","Elmhurst","Boro Zone"
83,"Queens","Elmhurst/Maspeth","Boro Zone"
84,"Staten Island","Eltingville/Annadale/Prince's Bay","Boro Zone"
85,"Brooklyn","Erasmus","Boro Zone"
86,"Queens","Far Rockaway","Boro Zone"
87,"Manhattan","Financial District North","Yellow Zone"
88,"Manhattan","Financial District South","Yellow Zone"
89,"Brooklyn","Flatbush/Ditmas Park","Boro Zone"
90,"Manhattan","Flatiron","Yellow Zone"
91,"Brooklyn","Flatlands","Boro Zone"
92,"Queens","Flushing","Boro Zone"
93,"Queens","Flushing Meadows-Corona Park","Boro Zone"
94,"Bronx","Fordham South","Boro Zone"
95,"Queens","Forest Hills","Boro Zone"
96,"Queens","Forest Park/Highland Park","Boro Zone"
97,"Brooklyn","Fort Greene","Boro Zone"
98,"Queens","Fresh Meadows","Boro Zone"
99,"Staten Island","Freshkills Park","Boro Zone"
100,"Manhattan","Garment District","Yellow Zone"
101,"Queens","Glen Oaks","Boro Zone"
102,"Queens","Glendale","Boro Zone"
103,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone"
104,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone"
105,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone"
106,"Brooklyn","Gowanus","Boro Zone"
107,"Manhattan","Gramercy","Yellow Zone"
108,"Brooklyn","Gravesend","Boro Zone"
109,"Staten Island","Great Kills","Boro Zone"
110,"Staten Island","Great Kills Park","Boro Zone"
111,"Brooklyn","Green-Wood Cemetery","Boro Zone"
112,"Brooklyn","Greenpoint","Boro Zone"
113,"Manhattan","Greenwich Village North","Yellow Zone"
114,"Manhattan","Greenwich Village South","Yellow Zone"
115,"Staten Island","Grymes Hill/Clifton","Boro Zone"
116,"Manhattan","Hamilton Heights","Boro Zone"
117,"Queens","Hammels/Arverne","Boro Zone"
118,"Staten Island","Heartland Village/Todt Hill","Boro Zone"
119,"Bronx","Highbridge","Boro Zone"
120,"Manhattan","Highbridge Park","Boro Zone"
121,"Queens","Hillcrest/Pomonok","Boro Zone"
122,"Queens","Hollis","Boro Zone"
123,"Brooklyn","Homecrest","Boro Zone"
124,"Queens","Howard Beach","Boro Zone"
125,"Manhattan","Hudson Sq","Yellow Zone"
126,"Bronx","Hunts Point","Boro Zone"
127,"Manhattan","Inwood","Boro Zone"
128,"Manhattan","Inwood Hill Park","Boro Zone"
129,"Queens","Jackson Heights","Boro Zone"
130,"Queens","Jamaica","Boro Zone"
131,"Queens","Jamaica Estates","Boro Zone"
132,"Queens","JFK Airport","Airports"
133,"Brooklyn","Kensington","Boro Zone"
134,"Queens","Kew Gardens","Boro Zone"
135,"Queens","Kew Gardens Hills","Boro Zone"
136,"Bronx","Kingsbridge Heights","Boro Zone"
137,"Manhattan","Kips Bay","Yellow Zone"
138,"Queens","LaGuardia Airport","Airports"
139,"Queens","Laurelton","Boro Zone"
140,"Manhattan","Lenox Hill East","Yellow Zone"
141,"Manhattan","Lenox Hill West","Yellow Zone"
142,"Manhattan","Lincoln Square East","Yellow Zone"
143,"Manhattan","Lincoln Square West","Yellow Zone"
144,"Manhattan","Little Italy/NoLiTa","Yellow Zone"
145,"Queens","Long Island City/Hunters Point","Boro Zone"
146,"Queens","Long Island City/Queens Plaza","Boro Zone"
147,"Bronx","Longwood","Boro Zone"
148,"Manhattan","Lower East Side","Yellow Zone"
149,"Brooklyn","Madison","Boro Zone"
150,"Brooklyn","Manhattan Beach","Boro Zone"
151,"Manhattan","Manhattan Valley","Yellow Zone"
152,"Manhattan","Manhattanville","Boro Zone"
153,"Manhattan","Marble Hill","Boro Zone"
154,"Brooklyn","Marine Park/Floyd Bennett Field","Boro Zone"
155,"Brooklyn","Marine Park/Mill Basin","Boro Zone"
156,"Staten Island","Mariners Harbor","Boro Zone"
157,"Queens","Maspeth","Boro Zone"
158,"Manhattan","Meatpacking/West Village West","Yellow Zone"
159,"Bronx","Melrose South","Boro Zone"
160,"Queens","Middle Village","Boro Zone"
161,"Manhattan","Midtown Center","Yellow Zone"
162,"Manhattan","Midtown East","Yellow Zone"
163,"Manhattan","Midtown North","Yellow Zone"
164,"Manhattan","Midtown South","Yellow Zone"
165,"Brooklyn","Midwood","Boro Zone"
166,"Manhattan","Morningside Heights","Boro Zone"
167,"Bronx","Morrisania/Melrose","Boro Zone"
168,"Bronx","Mott Haven/Port Morris","Boro Zone"
169,"Bronx","Mount Hope","Boro Zone"
170,"Manhattan","Murray Hill","Yellow Zone"
171,"Queens","Murray Hill-Queens","Boro Zone"
172,"Staten Island","New Dorp/Midland Beach","Boro Zone"
173,"Queens","North Corona","Boro Zone"
174,"Bronx","Norwood","Boro Zone"
175,"Queens","Oakland Gardens","Boro Zone"
176,"Staten Island","Oakwood","Boro Zone"
177,"Brooklyn","Ocean Hill","Boro Zone"
178,"Brooklyn","Ocean Parkway South","Boro Zone"
179,"Queens","Old Astoria","Boro Zone"
180,"Queens","Ozone Park","Boro Zone"
181,"Brooklyn","Park Slope","Boro Zone"
182,"Bronx","Parkchester","Boro Zone"
183,"Bronx","Pelham Bay","Boro Zone"
184,"Bronx","Pelham Bay Park","Boro Zone"
185,"Bronx","Pelham Parkway","Boro Zone"
186,"Manhattan","Penn Station/Madison Sq West","Yellow Zone"
187,"Staten Island","Port Richmond","Boro Zone"
188,"Brooklyn","Prospect-Lefferts Gardens","Boro Zone"
189,"Brooklyn","Prospect Heights","Boro Zone"
190,"Brooklyn","Prospect Park","Boro Zone"
191,"Queens","Queens Village","Boro Zone"
192,"Queens","Queensboro Hill","Boro Zone"
193,"Queens","Queensbridge/Ravenswood","Boro Zone"
194,"Manhattan","Randalls Island","Yellow Zone"
195,"Brooklyn","Red Hook","Boro Zone"
196,"Queens","Rego Park","Boro Zone"
197,"Queens","Richmond Hill","Boro Zone"
198,"Queens","Ridgewood","Boro Zone"
199,"Bronx","Rikers Island","Boro Zone"
200,"Bronx","Riverdale/North Riverdale/Fieldston","Boro Zone"
201,"Queens","Rockaway Park","Boro Zone"
202,"Manhattan","Roosevelt Island","Boro Zone"
203,"Queens","Rosedale","Boro Zone"
204,"Staten Island","Rossville/Woodrow","Boro Zone"
205,"Queens","Saint Albans","Boro Zone"
206,"Staten Island","Saint George/New Brighton","Boro Zone"
207,"Queens","Saint Michaels Cemetery/Woodside","Boro Zone"
208,"Bronx","Schuylerville/Edgewater Park","Boro Zone"
209,"Manhattan","Seaport","Yellow Zone"
210,"Brooklyn","Sheepshead Bay","Boro Zone"
211,"Manhattan","SoHo","Yellow Zone"
212,"Bronx","Soundview/Bruckner","Boro Zone"
213,"Bronx","Soundview/Castle Hill","Boro Zone"
214,"Staten Island","South Beach/Dongan Hills","Boro Zone"
215,"Queens","South Jamaica","Boro Zone"
216,"Queens","South Ozone Park","Boro Zone"
217,"Brooklyn","South Williamsburg","Boro Zone"
218,"Queens","Springfield Gardens North","Boro Zone"
219,"Queens","Springfield Gardens South","Boro Zone"
220,"Bronx","Spuyten Duyvil/Kingsbridge","Boro Zone"
221,"Staten Island","Stapleton","Boro Zone"
222,"Brooklyn","Starrett City","Boro Zone"
223,"Queens","Steinway","Boro Zone"
224,"Manhattan","Stuy Town/Peter Cooper Village","Yellow Zone"
225,"Brooklyn","Stuyvesant Heights","Boro Zone"
226,"Queens","Sunnyside","Boro Zone"
227,"Brooklyn","Sunset Park East","Boro Zone"
228,"Brooklyn","Sunset Park West","Boro Zone"
229,"Manhattan","Sutton Place/Turtle Bay North","Yellow Zone"
230,"Manhattan","Times Sq/Theatre District","Yellow Zone"
231,"Manhattan","TriBeCa/Civic Center","Yellow Zone"
232,"Manhattan","Two Bridges/Seward Park","Yellow Zone"
233,"Manhattan","UN/Turtle Bay South","Yellow Zone"
234,"Manhattan","Union Sq","Yellow Zone"
235,"Bronx","University Heights/Morris Heights","Boro Zone"
236,"Manhattan","Upper East Side North","Yellow Zone"
237,"Manhattan","Upper East Side South","Yellow Zone"
238,"Manhattan","Upper West Side North","Yellow Zone"
239,"Manhattan","Upper West Side South","Yellow Zone"
240,"Bronx","Van Cortlandt Park","Boro Zone"
241,"Bronx","Van Cortlandt Village","Boro Zone"
242,"Bronx","Van Nest/Morris Park","Boro Zone"
243,"Manhattan","Washington Heights North","Boro Zone"
244,"Manhattan","Washington Heights South","Boro Zone"
245,"Staten Island","West Brighton","Boro Zone"
246,"Manhattan","West Chelsea/Hudson Yards","Yellow Zone"
247,"Bronx","West Concourse","Boro Zone"
248,"Bronx","West Farms/Bronx River","Boro Zone"
249,"Manhattan","West Village","Yellow Zone"
250,"Bronx","Westchester Village/Unionport","Boro Zone"
251,"Staten Island","Westerleigh","Boro Zone"
252,"Queens","Whitestone","Boro Zone"
253,"Queens","Willets Point","Boro Zone"
254,"Bronx","Williamsbridge/Olinville","Boro Zone"
255,"Brooklyn","Williamsburg (North Side)","Boro Zone"
256,"Brooklyn","Williamsburg (South Side)","Boro Zone"
257,"Brooklyn","Windsor Terrace","Boro Zone"
258,"Queens","Woodhaven","Boro Zone"
259,"Bronx","Woodlawn/Wakefield","Boro Zone"
260,"Queens","Woodside","Boro Zone"
261,"Manhattan","World Trade Center","Yellow Zone"
262,"Manhattan","Yorkville East","Yellow Zone"
263,"Manhattan","Yorkville West","Yellow Zone"
264,"Unknown","NV","N/A"
265,"Unknown","NA","N/A"
"locationid","borough","zone","service_zone"
1,"EWR","Newark Airport","EWR"
2,"Queens","Jamaica Bay","Boro Zone"
3,"Bronx","Allerton/Pelham Gardens","Boro Zone"
4,"Manhattan","Alphabet City","Yellow Zone"
5,"Staten Island","Arden Heights","Boro Zone"
6,"Staten Island","Arrochar/Fort Wadsworth","Boro Zone"
7,"Queens","Astoria","Boro Zone"
8,"Queens","Astoria Park","Boro Zone"
9,"Queens","Auburndale","Boro Zone"
10,"Queens","Baisley Park","Boro Zone"
11,"Brooklyn","Bath Beach","Boro Zone"
12,"Manhattan","Battery Park","Yellow Zone"
13,"Manhattan","Battery Park City","Yellow Zone"
14,"Brooklyn","Bay Ridge","Boro Zone"
15,"Queens","Bay Terrace/Fort Totten","Boro Zone"
16,"Queens","Bayside","Boro Zone"
17,"Brooklyn","Bedford","Boro Zone"
18,"Bronx","Bedford Park","Boro Zone"
19,"Queens","Bellerose","Boro Zone"
20,"Bronx","Belmont","Boro Zone"
21,"Brooklyn","Bensonhurst East","Boro Zone"
22,"Brooklyn","Bensonhurst West","Boro Zone"
23,"Staten Island","Bloomfield/Emerson Hill","Boro Zone"
24,"Manhattan","Bloomingdale","Yellow Zone"
25,"Brooklyn","Boerum Hill","Boro Zone"
26,"Brooklyn","Borough Park","Boro Zone"
27,"Queens","Breezy Point/Fort Tilden/Riis Beach","Boro Zone"
28,"Queens","Briarwood/Jamaica Hills","Boro Zone"
29,"Brooklyn","Brighton Beach","Boro Zone"
30,"Queens","Broad Channel","Boro Zone"
31,"Bronx","Bronx Park","Boro Zone"
32,"Bronx","Bronxdale","Boro Zone"
33,"Brooklyn","Brooklyn Heights","Boro Zone"
34,"Brooklyn","Brooklyn Navy Yard","Boro Zone"
35,"Brooklyn","Brownsville","Boro Zone"
36,"Brooklyn","Bushwick North","Boro Zone"
37,"Brooklyn","Bushwick South","Boro Zone"
38,"Queens","Cambria Heights","Boro Zone"
39,"Brooklyn","Canarsie","Boro Zone"
40,"Brooklyn","Carroll Gardens","Boro Zone"
41,"Manhattan","Central Harlem","Boro Zone"
42,"Manhattan","Central Harlem North","Boro Zone"
43,"Manhattan","Central Park","Yellow Zone"
44,"Staten Island","Charleston/Tottenville","Boro Zone"
45,"Manhattan","Chinatown","Yellow Zone"
46,"Bronx","City Island","Boro Zone"
47,"Bronx","Claremont/Bathgate","Boro Zone"
48,"Manhattan","Clinton East","Yellow Zone"
49,"Brooklyn","Clinton Hill","Boro Zone"
50,"Manhattan","Clinton West","Yellow Zone"
51,"Bronx","Co-Op City","Boro Zone"
52,"Brooklyn","Cobble Hill","Boro Zone"
53,"Queens","College Point","Boro Zone"
54,"Brooklyn","Columbia Street","Boro Zone"
55,"Brooklyn","Coney Island","Boro Zone"
56,"Queens","Corona","Boro Zone"
57,"Queens","Corona","Boro Zone"
58,"Bronx","Country Club","Boro Zone"
59,"Bronx","Crotona Park","Boro Zone"
60,"Bronx","Crotona Park East","Boro Zone"
61,"Brooklyn","Crown Heights North","Boro Zone"
62,"Brooklyn","Crown Heights South","Boro Zone"
63,"Brooklyn","Cypress Hills","Boro Zone"
64,"Queens","Douglaston","Boro Zone"
65,"Brooklyn","Downtown Brooklyn/MetroTech","Boro Zone"
66,"Brooklyn","DUMBO/Vinegar Hill","Boro Zone"
67,"Brooklyn","Dyker Heights","Boro Zone"
68,"Manhattan","East Chelsea","Yellow Zone"
69,"Bronx","East Concourse/Concourse Village","Boro Zone"
70,"Queens","East Elmhurst","Boro Zone"
71,"Brooklyn","East Flatbush/Farragut","Boro Zone"
72,"Brooklyn","East Flatbush/Remsen Village","Boro Zone"
73,"Queens","East Flushing","Boro Zone"
74,"Manhattan","East Harlem North","Boro Zone"
75,"Manhattan","East Harlem South","Boro Zone"
76,"Brooklyn","East New York","Boro Zone"
77,"Brooklyn","East New York/Pennsylvania Avenue","Boro Zone"
78,"Bronx","East Tremont","Boro Zone"
79,"Manhattan","East Village","Yellow Zone"
80,"Brooklyn","East Williamsburg","Boro Zone"
81,"Bronx","Eastchester","Boro Zone"
82,"Queens","Elmhurst","Boro Zone"
83,"Queens","Elmhurst/Maspeth","Boro Zone"
84,"Staten Island","Eltingville/Annadale/Prince's Bay","Boro Zone"
85,"Brooklyn","Erasmus","Boro Zone"
86,"Queens","Far Rockaway","Boro Zone"
87,"Manhattan","Financial District North","Yellow Zone"
88,"Manhattan","Financial District South","Yellow Zone"
89,"Brooklyn","Flatbush/Ditmas Park","Boro Zone"
90,"Manhattan","Flatiron","Yellow Zone"
91,"Brooklyn","Flatlands","Boro Zone"
92,"Queens","Flushing","Boro Zone"
93,"Queens","Flushing Meadows-Corona Park","Boro Zone"
94,"Bronx","Fordham South","Boro Zone"
95,"Queens","Forest Hills","Boro Zone"
96,"Queens","Forest Park/Highland Park","Boro Zone"
97,"Brooklyn","Fort Greene","Boro Zone"
98,"Queens","Fresh Meadows","Boro Zone"
99,"Staten Island","Freshkills Park","Boro Zone"
100,"Manhattan","Garment District","Yellow Zone"
101,"Queens","Glen Oaks","Boro Zone"
102,"Queens","Glendale","Boro Zone"
103,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone"
104,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone"
105,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone"
106,"Brooklyn","Gowanus","Boro Zone"
107,"Manhattan","Gramercy","Yellow Zone"
108,"Brooklyn","Gravesend","Boro Zone"
109,"Staten Island","Great Kills","Boro Zone"
110,"Staten Island","Great Kills Park","Boro Zone"
111,"Brooklyn","Green-Wood Cemetery","Boro Zone"
112,"Brooklyn","Greenpoint","Boro Zone"
113,"Manhattan","Greenwich Village North","Yellow Zone"
114,"Manhattan","Greenwich Village South","Yellow Zone"
115,"Staten Island","Grymes Hill/Clifton","Boro Zone"
116,"Manhattan","Hamilton Heights","Boro Zone"
117,"Queens","Hammels/Arverne","Boro Zone"
118,"Staten Island","Heartland Village/Todt Hill","Boro Zone"
119,"Bronx","Highbridge","Boro Zone"
120,"Manhattan","Highbridge Park","Boro Zone"
121,"Queens","Hillcrest/Pomonok","Boro Zone"
122,"Queens","Hollis","Boro Zone"
123,"Brooklyn","Homecrest","Boro Zone"
124,"Queens","Howard Beach","Boro Zone"
125,"Manhattan","Hudson Sq","Yellow Zone"
126,"Bronx","Hunts Point","Boro Zone"
127,"Manhattan","Inwood","Boro Zone"
128,"Manhattan","Inwood Hill Park","Boro Zone"
129,"Queens","Jackson Heights","Boro Zone"
130,"Queens","Jamaica","Boro Zone"
131,"Queens","Jamaica Estates","Boro Zone"
132,"Queens","JFK Airport","Airports"
133,"Brooklyn","Kensington","Boro Zone"
134,"Queens","Kew Gardens","Boro Zone"
135,"Queens","Kew Gardens Hills","Boro Zone"
136,"Bronx","Kingsbridge Heights","Boro Zone"
137,"Manhattan","Kips Bay","Yellow Zone"
138,"Queens","LaGuardia Airport","Airports"
139,"Queens","Laurelton","Boro Zone"
140,"Manhattan","Lenox Hill East","Yellow Zone"
141,"Manhattan","Lenox Hill West","Yellow Zone"
142,"Manhattan","Lincoln Square East","Yellow Zone"
143,"Manhattan","Lincoln Square West","Yellow Zone"
144,"Manhattan","Little Italy/NoLiTa","Yellow Zone"
145,"Queens","Long Island City/Hunters Point","Boro Zone"
146,"Queens","Long Island City/Queens Plaza","Boro Zone"
147,"Bronx","Longwood","Boro Zone"
148,"Manhattan","Lower East Side","Yellow Zone"
149,"Brooklyn","Madison","Boro Zone"
150,"Brooklyn","Manhattan Beach","Boro Zone"
151,"Manhattan","Manhattan Valley","Yellow Zone"
152,"Manhattan","Manhattanville","Boro Zone"
153,"Manhattan","Marble Hill","Boro Zone"
154,"Brooklyn","Marine Park/Floyd Bennett Field","Boro Zone"
155,"Brooklyn","Marine Park/Mill Basin","Boro Zone"
156,"Staten Island","Mariners Harbor","Boro Zone"
157,"Queens","Maspeth","Boro Zone"
158,"Manhattan","Meatpacking/West Village West","Yellow Zone"
159,"Bronx","Melrose South","Boro Zone"
160,"Queens","Middle Village","Boro Zone"
161,"Manhattan","Midtown Center","Yellow Zone"
162,"Manhattan","Midtown East","Yellow Zone"
163,"Manhattan","Midtown North","Yellow Zone"
164,"Manhattan","Midtown South","Yellow Zone"
165,"Brooklyn","Midwood","Boro Zone"
166,"Manhattan","Morningside Heights","Boro Zone"
167,"Bronx","Morrisania/Melrose","Boro Zone"
168,"Bronx","Mott Haven/Port Morris","Boro Zone"
169,"Bronx","Mount Hope","Boro Zone"
170,"Manhattan","Murray Hill","Yellow Zone"
171,"Queens","Murray Hill-Queens","Boro Zone"
172,"Staten Island","New Dorp/Midland Beach","Boro Zone"
173,"Queens","North Corona","Boro Zone"
174,"Bronx","Norwood","Boro Zone"
175,"Queens","Oakland Gardens","Boro Zone"
176,"Staten Island","Oakwood","Boro Zone"
177,"Brooklyn","Ocean Hill","Boro Zone"
178,"Brooklyn","Ocean Parkway South","Boro Zone"
179,"Queens","Old Astoria","Boro Zone"
180,"Queens","Ozone Park","Boro Zone"
181,"Brooklyn","Park Slope","Boro Zone"
182,"Bronx","Parkchester","Boro Zone"
183,"Bronx","Pelham Bay","Boro Zone"
184,"Bronx","Pelham Bay Park","Boro Zone"
185,"Bronx","Pelham Parkway","Boro Zone"
186,"Manhattan","Penn Station/Madison Sq West","Yellow Zone"
187,"Staten Island","Port Richmond","Boro Zone"
188,"Brooklyn","Prospect-Lefferts Gardens","Boro Zone"
189,"Brooklyn","Prospect Heights","Boro Zone"
190,"Brooklyn","Prospect Park","Boro Zone"
191,"Queens","Queens Village","Boro Zone"
192,"Queens","Queensboro Hill","Boro Zone"
193,"Queens","Queensbridge/Ravenswood","Boro Zone"
194,"Manhattan","Randalls Island","Yellow Zone"
195,"Brooklyn","Red Hook","Boro Zone"
196,"Queens","Rego Park","Boro Zone"
197,"Queens","Richmond Hill","Boro Zone"
198,"Queens","Ridgewood","Boro Zone"
199,"Bronx","Rikers Island","Boro Zone"
200,"Bronx","Riverdale/North Riverdale/Fieldston","Boro Zone"
201,"Queens","Rockaway Park","Boro Zone"
202,"Manhattan","Roosevelt Island","Boro Zone"
203,"Queens","Rosedale","Boro Zone"
204,"Staten Island","Rossville/Woodrow","Boro Zone"
205,"Queens","Saint Albans","Boro Zone"
206,"Staten Island","Saint George/New Brighton","Boro Zone"
207,"Queens","Saint Michaels Cemetery/Woodside","Boro Zone"
208,"Bronx","Schuylerville/Edgewater Park","Boro Zone"
209,"Manhattan","Seaport","Yellow Zone"
210,"Brooklyn","Sheepshead Bay","Boro Zone"
211,"Manhattan","SoHo","Yellow Zone"
212,"Bronx","Soundview/Bruckner","Boro Zone"
213,"Bronx","Soundview/Castle Hill","Boro Zone"
214,"Staten Island","South Beach/Dongan Hills","Boro Zone"
215,"Queens","South Jamaica","Boro Zone"
216,"Queens","South Ozone Park","Boro Zone"
217,"Brooklyn","South Williamsburg","Boro Zone"
218,"Queens","Springfield Gardens North","Boro Zone"
219,"Queens","Springfield Gardens South","Boro Zone"
220,"Bronx","Spuyten Duyvil/Kingsbridge","Boro Zone"
221,"Staten Island","Stapleton","Boro Zone"
222,"Brooklyn","Starrett City","Boro Zone"
223,"Queens","Steinway","Boro Zone"
224,"Manhattan","Stuy Town/Peter Cooper Village","Yellow Zone"
225,"Brooklyn","Stuyvesant Heights","Boro Zone"
226,"Queens","Sunnyside","Boro Zone"
227,"Brooklyn","Sunset Park East","Boro Zone"
228,"Brooklyn","Sunset Park West","Boro Zone"
229,"Manhattan","Sutton Place/Turtle Bay North","Yellow Zone"
230,"Manhattan","Times Sq/Theatre District","Yellow Zone"
231,"Manhattan","TriBeCa/Civic Center","Yellow Zone"
232,"Manhattan","Two Bridges/Seward Park","Yellow Zone"
233,"Manhattan","UN/Turtle Bay South","Yellow Zone"
234,"Manhattan","Union Sq","Yellow Zone"
235,"Bronx","University Heights/Morris Heights","Boro Zone"
236,"Manhattan","Upper East Side North","Yellow Zone"
237,"Manhattan","Upper East Side South","Yellow Zone"
238,"Manhattan","Upper West Side North","Yellow Zone"
239,"Manhattan","Upper West Side South","Yellow Zone"
240,"Bronx","Van Cortlandt Park","Boro Zone"
241,"Bronx","Van Cortlandt Village","Boro Zone"
242,"Bronx","Van Nest/Morris Park","Boro Zone"
243,"Manhattan","Washington Heights North","Boro Zone"
244,"Manhattan","Washington Heights South","Boro Zone"
245,"Staten Island","West Brighton","Boro Zone"
246,"Manhattan","West Chelsea/Hudson Yards","Yellow Zone"
247,"Bronx","West Concourse","Boro Zone"
248,"Bronx","West Farms/Bronx River","Boro Zone"
249,"Manhattan","West Village","Yellow Zone"
250,"Bronx","Westchester Village/Unionport","Boro Zone"
251,"Staten Island","Westerleigh","Boro Zone"
252,"Queens","Whitestone","Boro Zone"
253,"Queens","Willets Point","Boro Zone"
254,"Bronx","Williamsbridge/Olinville","Boro Zone"
255,"Brooklyn","Williamsburg (North Side)","Boro Zone"
256,"Brooklyn","Williamsburg (South Side)","Boro Zone"
257,"Brooklyn","Windsor Terrace","Boro Zone"
258,"Queens","Woodhaven","Boro Zone"
259,"Bronx","Woodlawn/Wakefield","Boro Zone"
260,"Queens","Woodside","Boro Zone"
261,"Manhattan","World Trade Center","Yellow Zone"
262,"Manhattan","Yorkville East","Yellow Zone"
263,"Manhattan","Yorkville West","Yellow Zone"
264,"Unknown","NV","N/A"
265,"Unknown","NA","N/A"
1 locationid borough zone service_zone
2 1 EWR Newark Airport EWR
3 2 Queens Jamaica Bay Boro Zone
4 3 Bronx Allerton/Pelham Gardens Boro Zone
5 4 Manhattan Alphabet City Yellow Zone
6 5 Staten Island Arden Heights Boro Zone
7 6 Staten Island Arrochar/Fort Wadsworth Boro Zone
8 7 Queens Astoria Boro Zone
9 8 Queens Astoria Park Boro Zone
10 9 Queens Auburndale Boro Zone
11 10 Queens Baisley Park Boro Zone
12 11 Brooklyn Bath Beach Boro Zone
13 12 Manhattan Battery Park Yellow Zone
14 13 Manhattan Battery Park City Yellow Zone
15 14 Brooklyn Bay Ridge Boro Zone
16 15 Queens Bay Terrace/Fort Totten Boro Zone
17 16 Queens Bayside Boro Zone
18 17 Brooklyn Bedford Boro Zone
19 18 Bronx Bedford Park Boro Zone
20 19 Queens Bellerose Boro Zone
21 20 Bronx Belmont Boro Zone
22 21 Brooklyn Bensonhurst East Boro Zone
23 22 Brooklyn Bensonhurst West Boro Zone
24 23 Staten Island Bloomfield/Emerson Hill Boro Zone
25 24 Manhattan Bloomingdale Yellow Zone
26 25 Brooklyn Boerum Hill Boro Zone
27 26 Brooklyn Borough Park Boro Zone
28 27 Queens Breezy Point/Fort Tilden/Riis Beach Boro Zone
29 28 Queens Briarwood/Jamaica Hills Boro Zone
30 29 Brooklyn Brighton Beach Boro Zone
31 30 Queens Broad Channel Boro Zone
32 31 Bronx Bronx Park Boro Zone
33 32 Bronx Bronxdale Boro Zone
34 33 Brooklyn Brooklyn Heights Boro Zone
35 34 Brooklyn Brooklyn Navy Yard Boro Zone
36 35 Brooklyn Brownsville Boro Zone
37 36 Brooklyn Bushwick North Boro Zone
38 37 Brooklyn Bushwick South Boro Zone
39 38 Queens Cambria Heights Boro Zone
40 39 Brooklyn Canarsie Boro Zone
41 40 Brooklyn Carroll Gardens Boro Zone
42 41 Manhattan Central Harlem Boro Zone
43 42 Manhattan Central Harlem North Boro Zone
44 43 Manhattan Central Park Yellow Zone
45 44 Staten Island Charleston/Tottenville Boro Zone
46 45 Manhattan Chinatown Yellow Zone
47 46 Bronx City Island Boro Zone
48 47 Bronx Claremont/Bathgate Boro Zone
49 48 Manhattan Clinton East Yellow Zone
50 49 Brooklyn Clinton Hill Boro Zone
51 50 Manhattan Clinton West Yellow Zone
52 51 Bronx Co-Op City Boro Zone
53 52 Brooklyn Cobble Hill Boro Zone
54 53 Queens College Point Boro Zone
55 54 Brooklyn Columbia Street Boro Zone
56 55 Brooklyn Coney Island Boro Zone
57 56 Queens Corona Boro Zone
58 57 Queens Corona Boro Zone
59 58 Bronx Country Club Boro Zone
60 59 Bronx Crotona Park Boro Zone
61 60 Bronx Crotona Park East Boro Zone
62 61 Brooklyn Crown Heights North Boro Zone
63 62 Brooklyn Crown Heights South Boro Zone
64 63 Brooklyn Cypress Hills Boro Zone
65 64 Queens Douglaston Boro Zone
66 65 Brooklyn Downtown Brooklyn/MetroTech Boro Zone
67 66 Brooklyn DUMBO/Vinegar Hill Boro Zone
68 67 Brooklyn Dyker Heights Boro Zone
69 68 Manhattan East Chelsea Yellow Zone
70 69 Bronx East Concourse/Concourse Village Boro Zone
71 70 Queens East Elmhurst Boro Zone
72 71 Brooklyn East Flatbush/Farragut Boro Zone
73 72 Brooklyn East Flatbush/Remsen Village Boro Zone
74 73 Queens East Flushing Boro Zone
75 74 Manhattan East Harlem North Boro Zone
76 75 Manhattan East Harlem South Boro Zone
77 76 Brooklyn East New York Boro Zone
78 77 Brooklyn East New York/Pennsylvania Avenue Boro Zone
79 78 Bronx East Tremont Boro Zone
80 79 Manhattan East Village Yellow Zone
81 80 Brooklyn East Williamsburg Boro Zone
82 81 Bronx Eastchester Boro Zone
83 82 Queens Elmhurst Boro Zone
84 83 Queens Elmhurst/Maspeth Boro Zone
85 84 Staten Island Eltingville/Annadale/Prince's Bay Boro Zone
86 85 Brooklyn Erasmus Boro Zone
87 86 Queens Far Rockaway Boro Zone
88 87 Manhattan Financial District North Yellow Zone
89 88 Manhattan Financial District South Yellow Zone
90 89 Brooklyn Flatbush/Ditmas Park Boro Zone
91 90 Manhattan Flatiron Yellow Zone
92 91 Brooklyn Flatlands Boro Zone
93 92 Queens Flushing Boro Zone
94 93 Queens Flushing Meadows-Corona Park Boro Zone
95 94 Bronx Fordham South Boro Zone
96 95 Queens Forest Hills Boro Zone
97 96 Queens Forest Park/Highland Park Boro Zone
98 97 Brooklyn Fort Greene Boro Zone
99 98 Queens Fresh Meadows Boro Zone
100 99 Staten Island Freshkills Park Boro Zone
101 100 Manhattan Garment District Yellow Zone
102 101 Queens Glen Oaks Boro Zone
103 102 Queens Glendale Boro Zone
104 103 Manhattan Governor's Island/Ellis Island/Liberty Island Yellow Zone
105 104 Manhattan Governor's Island/Ellis Island/Liberty Island Yellow Zone
106 105 Manhattan Governor's Island/Ellis Island/Liberty Island Yellow Zone
107 106 Brooklyn Gowanus Boro Zone
108 107 Manhattan Gramercy Yellow Zone
109 108 Brooklyn Gravesend Boro Zone
110 109 Staten Island Great Kills Boro Zone
111 110 Staten Island Great Kills Park Boro Zone
112 111 Brooklyn Green-Wood Cemetery Boro Zone
113 112 Brooklyn Greenpoint Boro Zone
114 113 Manhattan Greenwich Village North Yellow Zone
115 114 Manhattan Greenwich Village South Yellow Zone
116 115 Staten Island Grymes Hill/Clifton Boro Zone
117 116 Manhattan Hamilton Heights Boro Zone
118 117 Queens Hammels/Arverne Boro Zone
119 118 Staten Island Heartland Village/Todt Hill Boro Zone
120 119 Bronx Highbridge Boro Zone
121 120 Manhattan Highbridge Park Boro Zone
122 121 Queens Hillcrest/Pomonok Boro Zone
123 122 Queens Hollis Boro Zone
124 123 Brooklyn Homecrest Boro Zone
125 124 Queens Howard Beach Boro Zone
126 125 Manhattan Hudson Sq Yellow Zone
127 126 Bronx Hunts Point Boro Zone
128 127 Manhattan Inwood Boro Zone
129 128 Manhattan Inwood Hill Park Boro Zone
130 129 Queens Jackson Heights Boro Zone
131 130 Queens Jamaica Boro Zone
132 131 Queens Jamaica Estates Boro Zone
133 132 Queens JFK Airport Airports
134 133 Brooklyn Kensington Boro Zone
135 134 Queens Kew Gardens Boro Zone
136 135 Queens Kew Gardens Hills Boro Zone
137 136 Bronx Kingsbridge Heights Boro Zone
138 137 Manhattan Kips Bay Yellow Zone
139 138 Queens LaGuardia Airport Airports
140 139 Queens Laurelton Boro Zone
141 140 Manhattan Lenox Hill East Yellow Zone
142 141 Manhattan Lenox Hill West Yellow Zone
143 142 Manhattan Lincoln Square East Yellow Zone
144 143 Manhattan Lincoln Square West Yellow Zone
145 144 Manhattan Little Italy/NoLiTa Yellow Zone
146 145 Queens Long Island City/Hunters Point Boro Zone
147 146 Queens Long Island City/Queens Plaza Boro Zone
148 147 Bronx Longwood Boro Zone
149 148 Manhattan Lower East Side Yellow Zone
150 149 Brooklyn Madison Boro Zone
151 150 Brooklyn Manhattan Beach Boro Zone
152 151 Manhattan Manhattan Valley Yellow Zone
153 152 Manhattan Manhattanville Boro Zone
154 153 Manhattan Marble Hill Boro Zone
155 154 Brooklyn Marine Park/Floyd Bennett Field Boro Zone
156 155 Brooklyn Marine Park/Mill Basin Boro Zone
157 156 Staten Island Mariners Harbor Boro Zone
158 157 Queens Maspeth Boro Zone
159 158 Manhattan Meatpacking/West Village West Yellow Zone
160 159 Bronx Melrose South Boro Zone
161 160 Queens Middle Village Boro Zone
162 161 Manhattan Midtown Center Yellow Zone
163 162 Manhattan Midtown East Yellow Zone
164 163 Manhattan Midtown North Yellow Zone
165 164 Manhattan Midtown South Yellow Zone
166 165 Brooklyn Midwood Boro Zone
167 166 Manhattan Morningside Heights Boro Zone
168 167 Bronx Morrisania/Melrose Boro Zone
169 168 Bronx Mott Haven/Port Morris Boro Zone
170 169 Bronx Mount Hope Boro Zone
171 170 Manhattan Murray Hill Yellow Zone
172 171 Queens Murray Hill-Queens Boro Zone
173 172 Staten Island New Dorp/Midland Beach Boro Zone
174 173 Queens North Corona Boro Zone
175 174 Bronx Norwood Boro Zone
176 175 Queens Oakland Gardens Boro Zone
177 176 Staten Island Oakwood Boro Zone
178 177 Brooklyn Ocean Hill Boro Zone
179 178 Brooklyn Ocean Parkway South Boro Zone
180 179 Queens Old Astoria Boro Zone
181 180 Queens Ozone Park Boro Zone
182 181 Brooklyn Park Slope Boro Zone
183 182 Bronx Parkchester Boro Zone
184 183 Bronx Pelham Bay Boro Zone
185 184 Bronx Pelham Bay Park Boro Zone
186 185 Bronx Pelham Parkway Boro Zone
187 186 Manhattan Penn Station/Madison Sq West Yellow Zone
188 187 Staten Island Port Richmond Boro Zone
189 188 Brooklyn Prospect-Lefferts Gardens Boro Zone
190 189 Brooklyn Prospect Heights Boro Zone
191 190 Brooklyn Prospect Park Boro Zone
192 191 Queens Queens Village Boro Zone
193 192 Queens Queensboro Hill Boro Zone
194 193 Queens Queensbridge/Ravenswood Boro Zone
195 194 Manhattan Randalls Island Yellow Zone
196 195 Brooklyn Red Hook Boro Zone
197 196 Queens Rego Park Boro Zone
198 197 Queens Richmond Hill Boro Zone
199 198 Queens Ridgewood Boro Zone
200 199 Bronx Rikers Island Boro Zone
201 200 Bronx Riverdale/North Riverdale/Fieldston Boro Zone
202 201 Queens Rockaway Park Boro Zone
203 202 Manhattan Roosevelt Island Boro Zone
204 203 Queens Rosedale Boro Zone
205 204 Staten Island Rossville/Woodrow Boro Zone
206 205 Queens Saint Albans Boro Zone
207 206 Staten Island Saint George/New Brighton Boro Zone
208 207 Queens Saint Michaels Cemetery/Woodside Boro Zone
209 208 Bronx Schuylerville/Edgewater Park Boro Zone
210 209 Manhattan Seaport Yellow Zone
211 210 Brooklyn Sheepshead Bay Boro Zone
212 211 Manhattan SoHo Yellow Zone
213 212 Bronx Soundview/Bruckner Boro Zone
214 213 Bronx Soundview/Castle Hill Boro Zone
215 214 Staten Island South Beach/Dongan Hills Boro Zone
216 215 Queens South Jamaica Boro Zone
217 216 Queens South Ozone Park Boro Zone
218 217 Brooklyn South Williamsburg Boro Zone
219 218 Queens Springfield Gardens North Boro Zone
220 219 Queens Springfield Gardens South Boro Zone
221 220 Bronx Spuyten Duyvil/Kingsbridge Boro Zone
222 221 Staten Island Stapleton Boro Zone
223 222 Brooklyn Starrett City Boro Zone
224 223 Queens Steinway Boro Zone
225 224 Manhattan Stuy Town/Peter Cooper Village Yellow Zone
226 225 Brooklyn Stuyvesant Heights Boro Zone
227 226 Queens Sunnyside Boro Zone
228 227 Brooklyn Sunset Park East Boro Zone
229 228 Brooklyn Sunset Park West Boro Zone
230 229 Manhattan Sutton Place/Turtle Bay North Yellow Zone
231 230 Manhattan Times Sq/Theatre District Yellow Zone
232 231 Manhattan TriBeCa/Civic Center Yellow Zone
233 232 Manhattan Two Bridges/Seward Park Yellow Zone
234 233 Manhattan UN/Turtle Bay South Yellow Zone
235 234 Manhattan Union Sq Yellow Zone
236 235 Bronx University Heights/Morris Heights Boro Zone
237 236 Manhattan Upper East Side North Yellow Zone
238 237 Manhattan Upper East Side South Yellow Zone
239 238 Manhattan Upper West Side North Yellow Zone
240 239 Manhattan Upper West Side South Yellow Zone
241 240 Bronx Van Cortlandt Park Boro Zone
242 241 Bronx Van Cortlandt Village Boro Zone
243 242 Bronx Van Nest/Morris Park Boro Zone
244 243 Manhattan Washington Heights North Boro Zone
245 244 Manhattan Washington Heights South Boro Zone
246 245 Staten Island West Brighton Boro Zone
247 246 Manhattan West Chelsea/Hudson Yards Yellow Zone
248 247 Bronx West Concourse Boro Zone
249 248 Bronx West Farms/Bronx River Boro Zone
250 249 Manhattan West Village Yellow Zone
251 250 Bronx Westchester Village/Unionport Boro Zone
252 251 Staten Island Westerleigh Boro Zone
253 252 Queens Whitestone Boro Zone
254 253 Queens Willets Point Boro Zone
255 254 Bronx Williamsbridge/Olinville Boro Zone
256 255 Brooklyn Williamsburg (North Side) Boro Zone
257 256 Brooklyn Williamsburg (South Side) Boro Zone
258 257 Brooklyn Windsor Terrace Boro Zone
259 258 Queens Woodhaven Boro Zone
260 259 Bronx Woodlawn/Wakefield Boro Zone
261 260 Queens Woodside Boro Zone
262 261 Manhattan World Trade Center Yellow Zone
263 262 Manhattan Yorkville East Yellow Zone
264 263 Manhattan Yorkville West Yellow Zone
265 264 Unknown NV N/A
266 265 Unknown NA N/A

21
04-analytics-engineering/taxi_rides_ny/dbt_project.yml → week_4_analytics_engineering/taxi_rides_ny/dbt_project.yml
View File

@ -7,13 +7,13 @@ version: '1.0.0'
config-version: 2
# This setting configures which "profile" dbt uses for this project.
profile: 'default'
profile: 'pg-dbt-workshop'
# These configurations specify where dbt should look for different types of files.
# The `model-paths` config, for example, states that models in this project can be
# The `source-paths` config, for example, states that models in this project can be
# found in the "models/" directory. You probably won't need to change these!
model-paths: ["models"]
analysis-paths: ["analyses"]
analysis-paths: ["analysis"]
test-paths: ["tests"]
seed-paths: ["seeds"]
macro-paths: ["macros"]
@ -21,20 +21,17 @@ snapshot-paths: ["snapshots"]
target-path: "target" # directory which will store compiled SQL files
clean-targets: # directories to be removed by `dbt clean`
- "target"
- "dbt_packages"
- "target"
- "dbt_packages"
- "dbt_modules"
# Configuring models
# Full documentation: https://docs.getdbt.com/docs/configuring-models
# In dbt, the default materialization for a model is a view. This means, when you run
# dbt run or dbt build, all of your models will be built as a view in your data platform.
# The configuration below will override this setting for models in the example folder to
# instead be materialized as tables. Any models you add to the root of the models folder will
# continue to be built as views. These settings can be overridden in the individual model files
# In this example config, we tell dbt to build all models in the example/ directory
# as tables. These settings can be overridden in the individual model files
# using the `{{ config(...) }}` macro.
models:
taxi_rides_ny:
# Applies to all files under models/.../
@ -49,4 +46,4 @@ seeds:
taxi_rides_ny:
taxi_zone_lookup:
+column_types:
locationid: numeric
locationid: numeric

0
04-analytics-engineering/taxi_rides_ny/macros/.gitkeep → week_4_analytics_engineering/taxi_rides_ny/macros/.gitkeep
View File

9
04-analytics-engineering/taxi_rides_ny/macros/get_payment_type_description.sql → week_4_analytics_engineering/taxi_rides_ny/macros/get_payment_type_description.sql
View File

@ -1,17 +1,18 @@
{#
{#
This macro returns the description of the payment_type
#}
{% macro get_payment_type_description(payment_type) -%}
case {{ dbt.safe_cast("payment_type", api.Column.translate_type("integer")) }}
case {{ payment_type }}
when 1 then 'Credit card'
when 2 then 'Cash'
when 3 then 'No charge'
when 4 then 'Dispute'
when 5 then 'Unknown'
when 6 then 'Voided trip'
else 'EMPTY'
end
{%- endmacro %}
{%- endmacro %}

0
04-analytics-engineering/taxi_rides_ny/macros/macros_properties.yml → week_4_analytics_engineering/taxi_rides_ny/macros/macros_properties.yml
View File

3
04-analytics-engineering/taxi_rides_ny/models/core/dim_zones.sql → week_4_analytics_engineering/taxi_rides_ny/models/core/dim_zones.sql
View File

@ -1,8 +1,9 @@
{{ config(materialized='table') }}
select
locationid,
borough,
zone,
replace(service_zone,'Boro','Green') as service_zone
replace(service_zone,'Boro','Green') as service_zone
from {{ ref('taxi_zone_lookup') }}

10
04-analytics-engineering/taxi_rides_ny/models/core/dm_monthly_zone_revenue.sql → week_4_analytics_engineering/taxi_rides_ny/models/core/dm_monthly_zone_revenue.sql
View File

@ -6,7 +6,8 @@ with trips_data as (
select
-- Reveneue grouping
pickup_zone as revenue_zone,
{{ dbt.date_trunc("month", "pickup_datetime") }} as revenue_month,
date_trunc('month', pickup_datetime) as revenue_month,
--Note: For BQ use instead: date_trunc(pickup_datetime, month) as revenue_month,
service_type,
@ -19,11 +20,12 @@ with trips_data as (
sum(ehail_fee) as revenue_monthly_ehail_fee,
sum(improvement_surcharge) as revenue_monthly_improvement_surcharge,
sum(total_amount) as revenue_monthly_total_amount,
sum(congestion_surcharge) as revenue_monthly_congestion_surcharge,
-- Additional calculations
count(tripid) as total_monthly_trips,
avg(passenger_count) as avg_monthly_passenger_count,
avg(trip_distance) as avg_monthly_trip_distance
avg(passenger_count) as avg_montly_passenger_count,
avg(trip_distance) as avg_montly_trip_distance
from trips_data
group by 1,2,3
group by 1,2,3

29
04-analytics-engineering/taxi_rides_ny/models/core/fact_trips.sql → week_4_analytics_engineering/taxi_rides_ny/models/core/fact_trips.sql
View File

@ -1,29 +1,29 @@
{{
config(
materialized='table'
)
}}
{{ config(materialized='table') }}
with green_tripdata as (
with green_data as (
select *,
'Green' as service_type
'Green' as service_type
from {{ ref('stg_green_tripdata') }}
),
yellow_tripdata as (
yellow_data as (
select *,
'Yellow' as service_type
from {{ ref('stg_yellow_tripdata') }}
),
trips_unioned as (
select * from green_tripdata
union all
select * from yellow_tripdata
select * from green_data
union all
select * from yellow_data
),
dim_zones as (
select * from {{ ref('dim_zones') }}
where borough != 'Unknown'
)
select trips_unioned.tripid,
select
trips_unioned.tripid,
trips_unioned.vendorid,
trips_unioned.service_type,
trips_unioned.ratecodeid,
@ -48,9 +48,10 @@ select trips_unioned.tripid,
trips_unioned.improvement_surcharge,
trips_unioned.total_amount,
trips_unioned.payment_type,
trips_unioned.payment_type_description
trips_unioned.payment_type_description,
trips_unioned.congestion_surcharge
from trips_unioned
inner join dim_zones as pickup_zone
on trips_unioned.pickup_locationid = pickup_zone.locationid
inner join dim_zones as dropoff_zone
on trips_unioned.dropoff_locationid = dropoff_zone.locationid
on trips_unioned.dropoff_locationid = dropoff_zone.locationid

26
week_4_analytics_engineering/taxi_rides_ny/models/core/schema.yml Normal file
View File

@ -0,0 +1,26 @@
version: 2
models:
- name: dim_zones
description: >
List of unique zones idefied by locationid.
Includes the service zone they correspond to (Green or yellow).
- name: fact_trips
description: >
Taxi trips corresponding to both service zones (Green and yellow).
The table contains records where both pickup and dropoff locations are valid and known zones.
Each record corresponds to a trip uniquely identified by tripid.
- name: dm_monthly_zone_revenue
description: >
Aggregated table of all taxi trips corresponding to both service zones (Green and yellow) per pickup zone, month and service.
The table contains monthly sums of the fare elements used to calculate the monthly revenue.
The table contains also monthly indicators like number of trips, and average trip distance.
columns:
- name: revenue_monthly_total_amount
description: Monthly sum of the the total_amount of the fare charged for the trip per pickup zone, month and service.
tests:
- not_null:
severity: error

24
04-analytics-engineering/taxi_rides_ny/models/staging/schema.yml → week_4_analytics_engineering/taxi_rides_ny/models/staging/schema.yml
View File

@ -1,16 +1,20 @@
version: 2
sources:
- name: staging
database: taxi-rides-ny-339813-412521
# For postgres:
#database: production
schema: trips_data_all
- name: staging
#For bigquery:
#database: taxi-rides-ny-339813
# For postgres:
database: production
schema: trips_data_all
# loaded_at_field: record_loaded_at
tables:
- name: green_tripdata
- name: yellow_tripdata
tables:
- name: green_tripdata
- name: yellow_tripdata
# freshness:
# error_after: {count: 6, period: hour}
@ -71,7 +75,7 @@ models:
memory before sending to the vendor, aka “store and forward,”
because the vehicle did not have a connection to the server.
Y= store and forward trip
N = not a store and forward trip
N= not a store and forward trip
- name: Dropoff_longitude
description: Longitude where the meter was disengaged.
- name: Dropoff_latitude
@ -196,4 +200,4 @@ models:
- name: Tolls_amount
description: Total amount of all tolls paid in trip.
- name: Total_amount
description: The total amount charged to passengers. Does not include cash tips.
description: The total amount charged to passengers. Does not include cash tips.

49
week_4_analytics_engineering/taxi_rides_ny/models/staging/stg_green_tripdata.sql Normal file
View File

@ -0,0 +1,49 @@
{{ config(materialized='view') }}
with tripdata as
(
select *,
row_number() over(partition by vendorid, lpep_pickup_datetime) as rn
from {{ source('staging','green_tripdata') }}
where vendorid is not null
)
select
-- identifiers
{{ dbt_utils.surrogate_key(['vendorid', 'lpep_pickup_datetime']) }} as tripid,
cast(vendorid as integer) as vendorid,
cast(ratecodeid as integer) as ratecodeid,
cast(pulocationid as integer) as pickup_locationid,
cast(dolocationid as integer) as dropoff_locationid,
-- timestamps
cast(lpep_pickup_datetime as timestamp) as pickup_datetime,
cast(lpep_dropoff_datetime as timestamp) as dropoff_datetime,
-- trip info
store_and_fwd_flag,
cast(passenger_count as integer) as passenger_count,
cast(trip_distance as numeric) as trip_distance,
cast(trip_type as integer) as trip_type,
-- payment info
cast(fare_amount as numeric) as fare_amount,
cast(extra as numeric) as extra,
cast(mta_tax as numeric) as mta_tax,
cast(tip_amount as numeric) as tip_amount,
cast(tolls_amount as numeric) as tolls_amount,
cast(ehail_fee as numeric) as ehail_fee,
cast(improvement_surcharge as numeric) as improvement_surcharge,
cast(total_amount as numeric) as total_amount,
cast(payment_type as integer) as payment_type,
{{ get_payment_type_description('payment_type') }} as payment_type_description,
cast(congestion_surcharge as numeric) as congestion_surcharge
from tripdata
where rn = 1
-- dbt build --m <model.sql> --var 'is_test_run: false'
{% if var('is_test_run', default=true) %}
limit 100
{% endif %}

24
04-analytics-engineering/taxi_rides_ny/models/staging/stg_yellow_tripdata.sql → week_4_analytics_engineering/taxi_rides_ny/models/staging/stg_yellow_tripdata.sql
View File

@ -9,19 +9,19 @@ with tripdata as
)
select
-- identifiers
{{ dbt_utils.generate_surrogate_key(['vendorid', 'tpep_pickup_datetime']) }} as tripid,
{{ dbt.safe_cast("vendorid", api.Column.translate_type("integer")) }} as vendorid,
{{ dbt.safe_cast("ratecodeid", api.Column.translate_type("integer")) }} as ratecodeid,
{{ dbt.safe_cast("pulocationid", api.Column.translate_type("integer")) }} as pickup_locationid,
{{ dbt.safe_cast("dolocationid", api.Column.translate_type("integer")) }} as dropoff_locationid,
{{ dbt_utils.surrogate_key(['vendorid', 'tpep_pickup_datetime']) }} as tripid,
cast(vendorid as integer) as vendorid,
cast(ratecodeid as integer) as ratecodeid,
cast(pulocationid as integer) as pickup_locationid,
cast(dolocationid as integer) as dropoff_locationid,
-- timestamps
cast(tpep_pickup_datetime as timestamp) as pickup_datetime,
cast(tpep_dropoff_datetime as timestamp) as dropoff_datetime,
-- trip info
store_and_fwd_flag,
{{ dbt.safe_cast("passenger_count", api.Column.translate_type("integer")) }} as passenger_count,
cast(passenger_count as integer) as passenger_count,
cast(trip_distance as numeric) as trip_distance,
-- yellow cabs are always street-hail
1 as trip_type,
@ -35,14 +35,16 @@ select
cast(0 as numeric) as ehail_fee,
cast(improvement_surcharge as numeric) as improvement_surcharge,
cast(total_amount as numeric) as total_amount,
coalesce({{ dbt.safe_cast("payment_type", api.Column.translate_type("integer")) }},0) as payment_type,
{{ get_payment_type_description('payment_type') }} as payment_type_description
cast(payment_type as integer) as payment_type,
{{ get_payment_type_description('payment_type') }} as payment_type_description,
cast(congestion_surcharge as numeric) as congestion_surcharge
from tripdata
where rn = 1
-- dbt build --select <model.sql> --vars '{'is_test_run: false}'
-- dbt build --m <model.sql> --var 'is_test_run: false'
{% if var('is_test_run', default=true) %}
limit 100
{% endif %}
{% endif %}

Some files were not shown because too many files have changed in this diff Show More