Commit 19d54784 authored by Hugo's avatar Hugo
Browse files

WIP II: adding shadow counters: overlaps, primary source periods

parent 006e9b81
......@@ -15,11 +15,11 @@ GEM
arel (9.0.0)
backports (3.11.4)
coderay (1.1.2)
concurrent-ruby (1.1.3)
daemons (1.2.6)
concurrent-ruby (1.1.4)
daemons (1.3.1)
diff-lcs (1.3)
eventmachine (1.2.7)
i18n (1.1.1)
i18n (1.5.3)
concurrent-ruby (~> 1.0)
method_source (0.9.2)
minitest (5.11.3)
......@@ -31,9 +31,9 @@ GEM
coderay (~> 1.1.0)
method_source (~> 0.9.0)
rack (2.0.6)
rack-protection (2.0.4)
rack-protection (2.0.5)
rack
rake (12.3.1)
rake (12.3.2)
rspec (3.8.0)
rspec-core (~> 3.8.0)
rspec-expectations (~> 3.8.0)
......@@ -47,18 +47,17 @@ GEM
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.8.0)
rspec-support (3.8.0)
sinatra (2.0.4)
sinatra (2.0.5)
mustermann (~> 1.0)
rack (~> 2.0)
rack-protection (= 2.0.4)
rack-protection (= 2.0.5)
tilt (~> 2.0)
sinatra-contrib (2.0.4)
activesupport (>= 4.0.0)
sinatra-contrib (2.0.5)
backports (>= 2.8.2)
multi_json
mustermann (~> 1.0)
rack-protection (= 2.0.4)
sinatra (= 2.0.4)
rack-protection (= 2.0.5)
sinatra (= 2.0.5)
tilt (>= 1.3, < 3)
sqlite3 (1.3.13)
thin (1.7.2)
......@@ -86,4 +85,4 @@ DEPENDENCIES
thin
BUNDLED WITH
1.17.1
2.0.1
......@@ -225,12 +225,6 @@ end
get '/graph_ui' do
require_logged_in
erb :graph_ui, :locals => {branches: Branch.all, selected_branch_id: params['branch_id'] }
......@@ -691,3 +685,128 @@ delete '/adm/api/schedule/:branch_id' do
{}.to_json
end
##########################
#### Overlap stuff ##
##########################
get '/adm/create_overlap' do
protected!
erb :create_overlap, layout: true,
:locals => {branches: Branch.all, counters: Counter.all}
end
get '/adm/edit_overlap/:id' do
protected!
selected_overlap = :id.present? ? Overlap.find(params[:id].to_i) : nil
selected_branch = selected_overlap.overlapping_counters.first.counter.branch
selected_counters = selected_overlap.overlapping_counters.map {|oc| oc.counter}
available_counters = selected_branch.counters - selected_counters
selected_overlap.get_missing_period_ranges
erb :edit_overlap, layout: true,
:locals => {overlaps: Overlap.all, selected_overlap: selected_overlap, overlap_id: :id, selected_branch: selected_branch,
selected_counters: selected_counters, available_counters: available_counters}
end
post '/adm/api/overlap' do
protected!
data = JSON.parse(request.body.read)
counter_ids = data['counter_ids'].to_a.map {|id| { counter_id: id.to_i}}
overlap = Overlap.create( name: data['name'], description: data['comment'], overlapping_counters_attributes: counter_ids)
if overlap.valid?
overlap.save!
{:success => true, :message => "OK", :overlap_id => overlap.id }.to_json
else
message = ''
overlap.errors.messages.each {|k,v| message += "* #{v} \n"}
{:success => false, :message => message}.to_json
end
end
delete '/adm/api/overlap/:overlap_id' do
protected!
overlap = Overlap.find(:overlap_id) rescue halt(400)
overlap.destroy!
end
post '/adm/api/overlap/:overlap_id/overlapping_counter/:counter_id' do
protected!
overlap = Overlap.find(params[:overlap_id].to_i) rescue halt(400)
overlap.add_counter!(params[:counter_id].to_i) rescue nil
success = overlap.errors.blank?
message = success ? "OK" : overlap.errors.messages.values
{:success => success, :message => message}.to_json
end
delete '/adm/api/overlap/:overlap_id/overlapping_counter/:overlapping_counter_id' do
protected!
overlap = Overlap.find(params[:overlap_id].to_i) rescue halt(400)
overlap.remove_counter!(params[:overlapping_counter_id].to_i) rescue nil
success = overlap.errors.blank?
message = success ? "OK" : overlap.errors.messages.values #.reduce('') {|str, val| str += val.class.name}
{:success => success, :message => message}.to_json
end
post '/adm/api/overlap/:overlap_id/primary_source_period' do
protected!
data = JSON.parse(request.body.read)
begin
counter_id = data['counter_id'].to_i
from_date = Date.strptime(data['from_date'], '%d-%m-%Y')
to_date = Date.strptime(data['to_date'], '%d-%m-%Y')
overlap = Overlap.find(params[:overlap_id].to_i)
rescue
halt(400)
end
overlap.add_primary_period!(counter_id, from_date, to_date) rescue nil
success = overlap.errors.blank?
message = success ? "OK" : overlap.errors.messages.values
{:success => success, :message => message}.to_json
end
delete '/adm/api/overlap/:overlap_id/primary_source_period/:period_id' do
puts "wha???"
protected!
overlap = Overlap.find(params[:overlap_id].to_i) rescue halt(400)
period_id = params[:period_id].to_i rescue halt(400)
overlap.remove_primary_period!(period_id)
success = overlap.errors.blank?
message = success ? "OK" : overlap.errors.messages.values
{:success => success, :message => message}.to_json
end
post '/adm/api/overlap/:overlap_id/simple_attribute' do
protected!
data = JSON.parse(request.body.read)
overlap = Overlap.find(params[:overlap_id].to_i) rescue halt(400)
halt 401, "Forbudt operasjon!" unless ['description', 'name'].include? data['attribute']
overlap[data['attribute']] = data['value']
overlap.save!(counter_id, from_date, to_date) rescue status 400
{}.to_json
end
......@@ -13,7 +13,6 @@ require_relative 'log'
ActiveRecord::Base.establish_connection(Settings::DB)
class DataCollector
def initialize
@logger = Log.new
......@@ -102,8 +101,10 @@ class DataCollector
visitors_in = row[:pedestrians_coming_in] || row[:in]
visitors_out = row[:pedestrians_going_out] || row[:out]
is_primary_source = counter.is_primary_source?(date)
record = Count.where(counter_id: counter.id, date: row_date, time: row_time).first_or_create
record.attributes = {visitors_in: visitors_in, visitors_out: visitors_out, oh_code: code}
record.attributes = {visitors_in: visitors_in, visitors_out: visitors_out, is_primary_source: is_primary_source, oh_code: code}
record.save! if record.changed?
end
rescue => e
......
......@@ -13,6 +13,9 @@ def dump(items)
end
end
accumulatedBranch = AccumulatedBranch.all
dump(accumulatedBranch)
branches = Branch.find([1,2])
dump(branches)
......@@ -25,3 +28,6 @@ dump(cameras)
primary_periods = counters.map {|counter| counter.primary_source_periods}.flatten
dump(primary_periods)
ct = counters[0].counts.limit(5)
dump(ct)
......@@ -53,9 +53,15 @@ ActiveRecord::Schema.define(version: 0) do
t.date "reopened_on", default: "2000-01-01"
end
create_table "counters", id: :integer, unsigned: true, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
create_table "counter_active_periods", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
t.integer "counter_id", null: false
t.date "period_begin", null: false
t.date "period_end"
end
create_table "counters", id: :integer, default: nil, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
t.integer "branch_id"
t.integer "source_id", unsigned: true
t.integer "source_id"
t.string "source_type"
t.boolean "is_active"
t.integer "source_type_id", null: false
......@@ -80,6 +86,13 @@ ActiveRecord::Schema.define(version: 0) do
t.string "name", limit: 30, null: false
end
create_table "metabranches", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
t.string "name", limit: 30, null: false
t.string "branch_code"
t.index ["branch_code"], name: "branch_code", unique: true
t.index ["name"], name: "name", unique: true
end
create_table "opening_hours", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
t.integer "weekday_id", limit: 2, null: false, unsigned: true
t.time "morning_start"
......@@ -92,8 +105,14 @@ ActiveRecord::Schema.define(version: 0) do
end
create_table "overlapping_counters", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
t.integer "counter_id_a", null: false
t.integer "counter_id_b", null: false
t.integer "overlap_id", null: false
t.integer "counter_id", null: false
t.string "description"
t.index ["overlap_id", "counter_id"], name: "id_overlap_counter", unique: true
end
create_table "overlaps", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
t.string "name", null: false
t.string "description"
end
......@@ -121,8 +140,11 @@ ActiveRecord::Schema.define(version: 0) do
create_table "primary_source_periods", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
t.integer "counter_id", null: false
t.date "begin", null: false
t.date "end"
t.integer "overlap_id", null: false
t.date "from_date", null: false
t.date "to_date"
t.index ["counter_id"], name: "fk_psp_counter"
t.index ["overlap_id"], name: "fk_psp_overlap"
end
create_table "source_types", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
......@@ -170,5 +192,11 @@ ActiveRecord::Schema.define(version: 0) do
t.timestamp "last_updated", default: -> { "CURRENT_TIMESTAMP" }, null: false
end
create_table "test", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
t.date "date"
end
add_foreign_key "counters", "branches", name: "fk_branch_id"
add_foreign_key "primary_source_periods", "counters", name: "primary_source_periods_ibfk_1"
add_foreign_key "primary_source_periods", "overlaps", name: "primary_source_periods_ibfk_2"
end
......@@ -48,9 +48,15 @@ ActiveRecord::Schema.define(version: 0) do
t.date "reopened_on", default: "2000-01-01"
end
create_table "counters", id: :integer, unsigned: true, force: :cascade do |t|
create_table "counter_active_periods", id: :integer, force: :cascade do |t|
t.integer "counter_id", null: false
t.date "period_begin", null: false
t.date "period_end"
end
create_table "counters", id: :integer, default: nil, force: :cascade do |t|
t.integer "branch_id"
t.integer "source_id", unsigned: true
t.integer "source_id"
t.string "source_type"
t.boolean "is_active"
t.integer "source_type_id", null: false
......@@ -72,6 +78,11 @@ ActiveRecord::Schema.define(version: 0) do
t.string "name", limit: 30, null: false
end
create_table "metabranches", id: :integer, force: :cascade do |t|
t.string "name", limit: 30, null: false
t.string "branch_code"
end
create_table "opening_hours", id: :integer, force: :cascade do |t|
t.integer "weekday_id", limit: 2, null: false, unsigned: true
t.time "morning_start"
......@@ -84,8 +95,13 @@ ActiveRecord::Schema.define(version: 0) do
end
create_table "overlapping_counters", id: :integer, force: :cascade do |t|
t.integer "counter_id_a", null: false
t.integer "counter_id_b", null: false
t.integer "overlap_id", null: false
t.integer "counter_id", null: false
t.string "description"
end
create_table "overlaps", id: :integer, force: :cascade do |t|
t.string "name", null: false
t.string "description"
end
......@@ -112,8 +128,9 @@ ActiveRecord::Schema.define(version: 0) do
create_table "primary_source_periods", id: :integer, force: :cascade do |t|
t.integer "counter_id", null: false
t.date "begin", null: false
t.date "end"
t.integer "overlap_id", null: false
t.date "from_date", null: false
t.date "to_date"
end
create_table "source_types", id: :integer, force: :cascade do |t|
......@@ -158,5 +175,11 @@ ActiveRecord::Schema.define(version: 0) do
t.timestamp "last_updated", default: -> { "CURRENT_TIMESTAMP" }, null: false
end
create_table "test", id: :integer, force: :cascade do |t|
t.date "date"
end
add_foreign_key "counters", "branches", name: "fk_branch_id"
add_foreign_key "primary_source_periods", "counters", name: "primary_source_periods_ibfk_1"
add_foreign_key "primary_source_periods", "overlaps", name: "primary_source_periods_ibfk_2"
end
......@@ -5,8 +5,10 @@ require 'time'
require 'active_record'
require 'active_support'
require 'json'
require 'set'
require_relative 'statistics'
require_relative 'overlaps'
require_relative '../graphmanager/graphmanager'
# under development....
......@@ -22,10 +24,7 @@ class SystemStatus < ActiveRecord::Base
end
# ----------------------------------------
# -------------------------------------------------------------------------------------------------
module BranchUtils
# Sums all counts from all counters in the given period.
......@@ -94,6 +93,7 @@ module BranchUtils
end
end
# -------------------------------------------------------------------------------------------------
class Branch < ActiveRecord::Base
include BranchUtils
......@@ -127,6 +127,10 @@ class Branch < ActiveRecord::Base
counts.where('(date = ? and time != ?) or (date = ? and time = ?) ', date, '00:00:00', date + 1, '00:00:00')
end
def get_primary_daily_counts(date)
counts.primary.where('(date = ? and time != ?) or (date = ? and time = ?) ', date, '00:00:00', date + 1, '00:00:00')
end
def find_first_active_date
counts.minimum(:date)
end
......@@ -254,13 +258,28 @@ class Branch < ActiveRecord::Base
anomalies
end
# TODO what does this do and who uses it???
def get_active_counts(start_date, end_date)
all_counts = counts.where('(date >= ? and date <= ?) or (date = ? and time = ?)', start_date, end_date, end_date + 1, '00:00:00')
end
def flag_primary_counts(from_date: find_first_active_date, to_date: find_last_active_date)
from_date..to_date.each do |date|
counters.each do |counter|
is_primary = counter.is_primary_source?(date)
counter.get_daily_counts(date).each do |count|
count.is_primary_source = is_primary
count.save if count.changed?
end
end
end
end
end
# -------------------------------------------------------------------------------------------------
# Meta entity for collecting accumulated stats from all the regular branches
# TODO ensure it handles primary counts only
class AccumulatedBranch < ActiveRecord::Base
include BranchUtils
......@@ -271,7 +290,7 @@ class AccumulatedBranch < ActiveRecord::Base
has_many :yearly_statistics, as: :aggregate
def counters
[]
[] # presenting all the subcounters would be messy, hence the empty list...
end
def has_active_oh?
......@@ -279,13 +298,18 @@ class AccumulatedBranch < ActiveRecord::Base
end
def counts
Count.all
Count.all # TODO: update to get primary counts only
end
def get_daily_counts(date)
# TODO don't think we'll ever need this one...
def get_daily_counts(date)
counts.where('(date = ? and time != ?) or (date = ? and time = ?) ', date, '00:00:00', date + 1, '00:00:00')
end
def get_primary_daily_counts(date)
counts.primary.where('(date = ? and time != ?) or (date = ? and time = ?) ', date, '00:00:00', date + 1, '00:00:00')
end
def find_first_active_date
Count.minimum(:date)
......@@ -301,30 +325,7 @@ class AccumulatedBranch < ActiveRecord::Base
end
end
class PrimarySourcePeriod < ActiveRecord::Base
# TODO find a better name for this one....
def self.is_primary_source(periods, period_start, period_end = nil)
period_end = period_start if period_end.nil?
total_no_of_days = (period_start..period_end).count
no_of_hits = 0
(period_start..period_end).each do |date|
periods.each do |period|
no_of_hits += 1 if period.begin <= date && period.end >= date
end
end
(no_of_hits*100)/total_no_of_days
end
end
class OverlappingCounter < ActiveRecord::Base
end
# -------------------------------------------------------------------------------------------------
class Counter < ActiveRecord::Base
has_many :counts
......@@ -332,6 +333,10 @@ class Counter < ActiveRecord::Base
belongs_to :branch
belongs_to :source, :polymorphic => true
#has_many :overlaps
has_many :overlapping_counters #, inverse_of: :counter
has_many :overlaps, through: :overlapping_counters
has_many :primary_source_periods
has_many :daily_statistics, as: :aggregate
......@@ -356,15 +361,35 @@ class Counter < ActiveRecord::Base
counts.where('(date = ? and time != ?) or (date = ? and time = ?) ', date, '00:00:00', date + 1, '00:00:00')
end
def get_primary_daily_counts(date)
counts.primary.where('(date = ? and time != ?) or (date = ? and time = ?) ', date, '00:00:00', date + 1, '00:00:00')
end
# this is ever-so-slightly inaccurate, but fix is not worth the overhead
def get_counts(first, last)
counts.where('(date >= ?) and (date <= ?) ', first, last)
end
# TODO def get_primary_count
def get_primary_counts(first, last)
counts.primary.where('(date >= ?) and (date <= ?) ', first, last)
end
def is_primary_source(period_start, period_end = nil)
PrimarySourcePeriod.is_primary_source(primary_source_periods, period_start, period_end)
def is_part_of_overlap?
# TODO what if it's part of several overlaps?
overlaps.present?
end
def is_primary_source?(date)
status = primary_source_periods.blank?
primary_source_periods.each do |period|
if (period.from_date..period.to_date).to_a.include?(date)
status = true
break
end
end
status
end
def find_first_active_date
......@@ -372,7 +397,7 @@ class Counter < ActiveRecord::Base
end
def find_last_active_date
last_active_date || Date.today #counts.maximum(:date)
last_active_date || Date.today
end
def get_recent_camera_status
......@@ -465,27 +490,15 @@ class Counter < ActiveRecord::Base
end
class CounterActivePeriod < ActiveRecord::Base
# period_begin <= period_end
# get_first_active_date
# get_last_active_date
# counter - get_active_dates
end
# -------------------------------------------------------------------------------------------------
class SourceType < ActiveRecord::Base
end
# -------------------------------------------------------------------------------------------------
class CameraInterface < ActiveRecord::Base
end
# -------------------------------------------------------------------------------------------------
class Camera < ActiveRecord::Base
has_one :counter, :as => :source
belongs_to :camera_interface
......@@ -541,9 +554,7 @@ class Camera < ActiveRecord::Base
end
# ----------
# -------------------------------------------------------------------------------------------------
class Portal < ActiveRecord::Base
has_one :counter, :as => :source
......@@ -598,195 +609,187 @@ class Portal < ActiveRecord::Base
}
}
}
})
response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: false) do |http|
http.request(request)
end
# massage results into CSV
es_results = JSON.parse(response.body)
buckets = es_results["aggregations"]["visitors_total"]["action"]["buckets"]
counts = {}
buckets.each do |bucket|
type = bucket["key"]
series = bucket["visitors_over_time"]
series["buckets"].each do |serie|
timestamp = serie["key_as_string"]
if counts[timestamp]
counts[timestamp].merge!({type => serie["doc_count"].to_s})
else
counts[timestamp] = {type => serie["doc_count"].to_s}
end
end
end