pad table with zeroes and add last update info

.gitignore

@@ -4,3 +4,4 @@ __pycache__
 .venv
 *jpg
 data.csv
+danedzis

README.md

@@ -9,7 +9,7 @@ pip3 install -r requirements.txt
 sudo apt install chromium
 chromium --version
 ```
-Then download the appropriate version of `chromedriver` from [this webpage](https://chromedriver.chromium.org/downloads), extract it, give execution permissions and put it in .venv/bin/. Here I'm downloading chromedriver for chromium 112
+Then download the appropriate version of `chromedriver` from [this webpage](https://chromedriver.chromium.org/downloads), extract it, give execution permissions and put it in '.venv/bin/'. Here I'm downloading chromedriver for chromium 112
 ```
 wget "https://chromedriver.storage.googleapis.com/112.0.5615.49/chromedriver_linux64.zip"
 unzip chromedriver_linux64.zip
@@ -18,5 +18,11 @@ chmod +x .venv/bin/chromedriver
 rm LICENSE.chromedriver chromedriver_linux64.zip
 ```
 
+What I do is run 2 daily cronjobs: one to get the data and another one to generate graphs:
+```
+45 11   * * *   cd /var/www/olx-visualiser/ && .venv/bin/python3 main.py
+47 11   * * *   cd /var/www/olx-visualiser/ && .venv/bin/python3 plot.py
+```
+
 `main.py` puts data in the database in the `olx_data` table.
 

index.html

@@ -1,6 +1,5 @@
 <h1 id="mieszkania-na-olx">Mieszkania na OLX</h1>
-<p><a href="olx_data.db">pobierz baze danych sqlite</a></p>
-<p><a href="data.csv">pobierz dane do wykresow (csv)</a></p>
+<embed src="danedzis">
 <figure>
 <img src="week.jpg" alt="" /><figcaption>wykres z ostatniego tygodnia</figcaption>
 </figure>
@@ -8,5 +7,7 @@
 <img src="month.jpg" alt="" /><figcaption>wykres z ostatniego miesiaca</figcaption>
 </figure>
 <figure>
-<img src="threemonth.jpg" alt="" /><figcaption>wykres z ostatnich 3 miesiecy</figcaption>
+<img src="threemonths.jpg" alt="" /><figcaption>wykres z ostatnich 3 miesiecy</figcaption>
+<p><a href="olx_data.db">pobierz baze danych sqlite</a></p>
+<p><a href="data.csv">pobierz dane do wykresow (csv)</a></p>
 </figure>

main.py

@@ -39,6 +39,8 @@ for line in cities_file:
     if line[0] != "#":
         cities.append(json.loads(line))
 
+dane_dzis = open("danedzis", "w")
+
 url_base = 'https://www.olx.pl/d/nieruchomosci/mieszkania/'
 cities_list = [(url_base + unidecode(i["name"]).lower().replace(" ", "-") + i["is_dubble"] + '/', i["city_id"]) for i in cities]
 not_empty = len(cities_list)
@@ -59,6 +61,8 @@ for city in cities:
                             VALUES
                                 (?, ?, ?, ?, ?); """
             con.insert_data(query, values)
+            dane_dzis.write("ostatnia aktualizacja: %s\nogloszen sprzedazy: %d\nogloszen wynajmu: %d\n" % (values[4], values[1], values[0]))
+            dane_dzis.close()
 #        except Exception as e:
 #            logger.info(f'an exception has occurred:{e}')
 #            pass

plot.py

@@ -1,7 +1,8 @@
 #!.venv/bin/python
 from db_con import DbConnect
 from matplotlib import pyplot as plt
-from datetime import date
+from matplotlib import ticker
+from datetime import date, timedelta
 import csv
 
 con = DbConnect("olx_data.db")
@@ -15,48 +16,59 @@ data = con.get_data(query)
 dates = []
 rent = []
 sell = []
-exchange = []
+#exchange = []
 
 for i in data:
     sell.append(i[3])
     rent.append(i[4])
-    exchange.append(i[5])
+#    exchange.append(i[5])
     dates.append(date.fromisoformat(i[6]))
 
-if len(dates) >= 7:
-    plt.plot(dates[-7:], rent[-7:], label="wynajem")
-    plt.plot(dates[-7:], sell[-7:], label="sprzedaż")
-    plt.plot(dates[-7:], exchange[-7:], label="zamiana")
-    plt.xlabel("data")
-    plt.xticks(rotation=45)
-    plt.ylabel("ilość ogłoszeń")
-    plt.title("ogłoszenia o mieszkaniach na olx.pl (ostatnie 7 dni)")
-    plt.legend()
-    plt.savefig("week.jpg", bbox_inches="tight", dpi=150)
-    plt.cla()
+firstday = dates[0]
+dates = [firstday-timedelta(days=x) for x in range(91 - len(dates), 0, -1)] + dates
+sell = [0] * (90-len(sell)) + sell
+rent = [0] * (90-len(rent)) + rent
+#exchange = [0] * (90-len(exchange)) + exchange
 
-if len(dates) >= 30:
-    plt.plot(dates[-30:], rent[-30:], label="wynajem")
-    plt.plot(dates[-30:], sell[-30:], label="sprzedaż")
-    plt.plot(dates[-30:], exchange[-30:], label="zamiana")
-    plt.xlabel("data")
-    plt.xticks(rotation=45)
-    plt.ylabel("ilość ogłoszeń")
-    plt.title("ogłoszenia o mieszkaniach na olx.pl (ostatnie 30 dni)")
-    plt.legend()
-    plt.savefig("month.jpg", bbox_inches="tight", dpi=150)
-    plt.cla()
+plt.plot(dates[-7:], rent[-7:], label="wynajem")
+plt.plot(dates[-7:], sell[-7:], label="sprzedaż")
+plt.xlabel("data")
+plt.xticks(rotation=45)
+plt.ylim(sorted(set(sell+rent))[1]-200, max(sell+rent)+200)
+plt.ylabel("ilość ogłoszeń")
+plt.title("ogłoszenia o mieszkaniach na olx.pl (ostatnie 7 dni)")
+plt.gca().yaxis.set_major_locator(plt.MultipleLocator(200))
+plt.legend()
+plt.grid()
+plt.savefig("week.jpg", bbox_inches="tight", dpi=150)
+plt.cla()
 
-if len(dates) >= 90:
-    threemonths.plot(dates[-90:], rent[-90:], label="wynajem")
-    threemonths.plot(dates[-90:], sell[-90:], label="sprzedaż")
-    threemonths.plot(dates[-90:], exchange[-90:], label="zamiana")
-    threemonths.xlabel("data")
-    threemonths.xticks(rotation=45)
-    threemonths.ylabel("ilość ogłoszeń")
-    threemonths.title("ogłoszenia o mieszkaniach na olx.pl (ostatnie 90 dni)")
-    threemonths.legend()
-    threemonths.savefig("threemonths.jpg", bbox_inches="tight", dpi=150)
+plt.plot(dates[-30:], rent[-30:], label="wynajem")
+plt.plot(dates[-30:], sell[-30:], label="sprzedaż")
+#plt.plot(dates[-30:], exchange[-30:], label="zamiana")
+plt.xlabel("data")
+plt.xticks(rotation=45)
+plt.ylabel("ilość ogłoszeń")
+plt.title("ogłoszenia o mieszkaniach na olx.pl (ostatnie 30 dni)")
+plt.ylim(sorted(set(sell+rent))[1]-200, max(sell+rent)+200)
+plt.gca().yaxis.set_major_locator(plt.MultipleLocator(200))
+plt.legend()
+plt.grid()
+plt.savefig("month.jpg", bbox_inches="tight", dpi=150)
+plt.cla()
+
+plt.plot(dates[-90:], rent[-90:], label="wynajem")
+plt.plot(dates[-90:], sell[-90:], label="sprzedaż")
+#plt.plot(dates[-90:], exchange[-90:], label="zamiana")
+plt.xlabel("data")
+plt.xticks(rotation=45)
+plt.ylabel("ilość ogłoszeń")
+plt.title("ogłoszenia o mieszkaniach na olx.pl (ostatnie 90 dni)")
+plt.ylim(sorted(set(sell+rent))[1]-200, max(sell+rent)+200)
+plt.gca().yaxis.set_major_locator(plt.MultipleLocator(200))
+plt.legend()
+plt.grid()
+plt.savefig("threemonths.jpg", bbox_inches="tight", dpi=150)
 
 cursor=con.connection.cursor()
 cursor.execute("SELECT * FROM olx_data;")