#!usr/bin/perl
use strict;
use warnings;
no warnings 'redefine';
# use diagnostics;
use Data::Dumper;
use HTML::TreeBuilder::XPath;
use feature qw(say);
use File::Basename;
use HTTP::Cookies;
use MIME::Base64 qw(decode_base64);
use Encode;
use utf8;
use DateTime::Format::Strptime;
use DateTime::Format::MySQL;
use DBI;
use DBD::MariaDB;
use DateTime::Format::DateParse;
use autodie qw(:all);
use Parallel::ForkManager;
use DBIx::Connector;
# use Storable;
# use Data::Printer; 
# use DateTime;
# use HTML::Entities;
# binmode STDOUT, ':encoding(utf-8)';
# binmode STDIN, ':encoding(utf-8)';
# use open ':std', ':encoding(UTF-8)';

my ($zsmappointment, $ts, $mech, $ua, $cookie_jar, $strp, %data, %services, %venues, $connection, $strp_datetime, $pm);

my %config = (
    'MYSQL' => {
            'ACCESS' => {
                'database'  => 'terminsnipe',
                'hostname'  => 'zinomedia.de',
                'user' => 'terminsnipe',
                'pw'  => 'dIHB6JLk0CBBx1p7',
                'port'  => 3306,
            },
            'OPTIONS' => {
                'PrintError' => 0, 
                'RaiseError' => 1,
                'AutoCommit' => 1,
            }
                },
    'URL_ROOT' => 'https://service.berlin.de',
    'URL_TERMIN_DAY' => 'https://service.berlin.de/terminvereinbarung/termin/day',
    'URL_TERMIN_TIME' => 'https://service.berlin.de/terminvereinbarung/termin/time',
    'URL_HUMAN' => 'https://service.berlin.de/terminvereinbarung/termin/human',
    'URL_STANDORTE' => 'https://service.berlin.de/standorte',
    'URL_DIENSTLEISTUNGEN' => 'https://service.berlin.de/dienstleistungen',
    'BROWSER' => 'MECH',
    'CAPTCHA_DIR' => './captcha',
    'COOKIE_DIR' => './cookie',
    'COOKIE_PATH' => './cookie/.cookies.txt',
    'FOLLOW_NEXT' => 1,
    'OUTPUT_PATH' => './output/data',
    'FILTER_SERVICES_TO_OUTPUT' => 1,
    'FILTER_SERVICES_PATH' => './output/filter_services',
    'DB_REFRESH_SERVICES' => 0,
    'RUN_SERVICES' => 0,
    'RUN_VENUES' => 0,
    'VENUES_TO_FILE' => 1,
    'VENUES_PATH' => './output/venues',
    'DB_REFRESH_VENUES' => 0,
    'DUMP_SERVICES_TO_FILE' => 1,
    'SERVICES_PATH' => './output/services',
    'OUTPUT_DIR' => './output',
    'DB_CLEAN_DATETIMES' => 1,
    'RATE_LIMIT_SLEEP_MIN' => 5,
    'SLEEP_LOOP_MIN' => 1, 
    'SERVICES' => {
            'Melderegisterauskunft sperren' => { 'RUN' => 0, 'ID' => 120678 },
            'Erweiterung einer Fahrerlaubnis' => { 'RUN' => 0, 'ID' => 121629 },
            'Ersatzführerschein nach Verlust/Diebstahl' => { 'RUN' => 0, 'ID' => 121593 },
            'Kinderreisepass beantragen / verlängern / aktualisieren' => { 'RUN' => 0, 'ID' => 121469 },
            'Namensänderung in den Fahrzeugpapieren' => { 'RUN' => 0, 'ID' => 324173 },
            'Änderung/Wechsel der Hauptwohnung' => { 'RUN' => 0, 'ID' => 120697 },
            'Umschreibung einer ausländischen Fahrerlaubnis aus einem Nicht-EU/EWR-Land (Drittstaat/Anlage 11)' => { 'RUN' => 0, 'ID' => 327537 },
            'Historisches Kennzeichen beantragen' => { 'RUN' => 0, 'ID' => 121478 },
            'Verlängerung der Fahrerlaubnis' => { 'RUN' => 0, 'ID' => 121634 },
            'Adressänderung in der Zulassungsbescheinigung Teil I (ZBI) bzw. in dem Fahrzeugschein' => { 'RUN' => 0, 'ID' => 120658 },
            'Blaue Karte EU zu einem neuen Pass übertragen' => { 'RUN' => 0, 'ID' => 326798 },
            'Neufahrzeug anmelden' => { 'RUN' => 0, 'ID' => 120882 },
            'Zulassung eines aus dem EU-Ausland eingeführten fabrikneuen Fahrzeuges mit Zulassungsbescheinigung Teil II' => { 'RUN' => 0, 'ID' => 324199 },
            'Führungszeugnis' => { 'RUN' => 0, 'ID' => 120926 },
            'Aufenthaltserlaubnis für im Bundesgebiet geborene Kinder - Erteilung' => { 'RUN' => 0, 'ID' => 324269 },
            'Kraftfahrzeug – technische Änderung melden' => { 'RUN' => 0, 'ID' => 120904 },
            'Reisepass beantragen' => { 'RUN' => 0, 'ID' => 121151 },
            'Meldebescheinigung beantragen' => { 'RUN' => 0, 'ID' => 120702 },
            'Umtausch eines Kartenführerscheins' => { 'RUN' => 0, 'ID' => 121616 },
            'Umschreibung einer Dienstfahrerlaubnis' => { 'RUN' => 0, 'ID' => 121615 },
            'Bescheinigung über ein unbefristetes Aufenthaltsrecht' => { 'RUN' => 0, 'ID' => 324921 },
            'Elektrokennzeichen beantragen' => { 'RUN' => 0, 'ID' => 327084 },
            'Abmeldung einer Wohnung' => { 'RUN' => 0, 'ID' => 120335 },
            'Zulassungsbescheinigung Teil II für KFZ wegen Verlust oder Diebstahl ersetzen' => { 'RUN' => 0, 'ID' => 121584 },
            'Beglaubigung von Unterschriften' => { 'RUN' => 0, 'ID' => 158142 },
            'Kraftfahrzeugkennzeichen wegen Unleserlichkeit ersetzen' => { 'RUN' => 0, 'ID' => 324907 },
            'Zulassung von Taxen (Kraftdroschken)' => { 'RUN' => 0, 'ID' => 326033 },
            'Fahrerlaubnis - Erweiterung auf die Klassen D1, D1E, D und DE' => { 'RUN' => 0, 'ID' => 324450 },
            'Wiederzulassung eines Kraftfahrzeuges ohne Halterwechsel beantragen' => { 'RUN' => 0, 'ID' => 120905 },
            'Ersterteilung einer Fahrerlaubnis' => { 'RUN' => 0, 'ID' => 121627 },
            'Beglaubigung von Kopien' => { 'RUN' => 0, 'ID' => 121701 },
            'Saisonkennzeichen für KFZ beantragen' => { 'RUN' => 0, 'ID' => 121480 },
            'Anmeldung einer Wohnung' => { 'RUN' => 0, 'ID' => 120686 },
            'Zulassung eines aus dem EU-Ausland eingeführten fabrikneuen Fahrzeuges ohne Zulassungsbescheinigung Teil II' => { 'RUN' => 0, 'ID' => 324200 },
            'Kraftfahrzeug ummelden – nach einem Umzug nach Berlin' => { 'RUN' => 0, 'ID' => 120918 },
            'Umschreibung eines zugelassenen Fahrzeuges mit Berliner Kennzeichen auf einen anderen Halter' => { 'RUN' => 0, 'ID' => 120906 },
            'Reisepass beantragen (vorläufiger Reisepass)' => { 'RUN' => 0, 'ID' => 121153 },
            'Kraftfahrzeug außer Betrieb setzen (Kfz abmelden)' => { 'RUN' => 0, 'ID' => 120877 },
            'Umstellung der Fahrerlaubnis' => { 'RUN' => 0, 'ID' => 124556 },
            'Widerspruchsrechte gegen Datenübermittlungen und Melderegisterauskünfte' => { 'RUN' => 0, 'ID' => 319141 },
            'Gewerbezentralregister - Auskunft beantragen' => { 'RUN' => 0, 'ID' => 327835 },
            'Wechselkennzeichen beantragen' => { 'RUN' => 0, 'ID' => 324587 },
            'Personalausweis vorläufig beantragen' => { 'RUN' => 0, 'ID' => 120682 },
            'Personalausweis beantragen' => { 'RUN' => 0, 'ID' => 120703 },
            'Kraftfahrzeugkennzeichen ersetzen – nach Diebstahl oder Verlust eines Nummernschildes' => { 'RUN' => 0, 'ID' => 324196 },
            'Umschreibung einer ausländischen Fahrerlaubnis aus einem EU-/EWR-Staat' => { 'RUN' => 0, 'ID' => 121598 },
            'Selbstfahrermietfahrzeug – Beginn/Beendigung der Nutzung' => { 'RUN' => 0, 'ID' => 326039 },
            'Zulassungsbescheinigung Teil I für KFZ wegen Verlust oder Diebstahl ersetzen' => { 'RUN' => 0, 'ID' => 121575 },
            'Kraftfahrzeugkennzeichen auf Wunsch ändern' => { 'RUN' => 0, 'ID' => 121473 },
            'Begleitetes Fahren mit 17' => { 'RUN' => 0, 'ID' => 121589 },
            'Fahrerlaubnis zur Fahrgastbeförderung (P-Schein) - Verlängerung' => { 'RUN' => 0, 'ID' => 324389 },
            'Zulassung eines außer Betrieb gesetzten Fahrzeuges mit Berliner Kennzeichen auf einen anderen Halter' => { 'RUN' => 0, 'ID' => 324169 },
            'Befreiung von der Ausweispflicht' => { 'RUN' => 0, 'ID' => 327044 },
            'Grünes Kennzeichen beantragen' => { 'RUN' => 0, 'ID' => 121482 },
            'Aufenthaltserlaubnis in einen neuen Pass übertragen' => { 'RUN' => 0, 'ID' => 121874 },
            'Kraftfahrzeug außer Betrieb setzen, unvollständige Unterlagen (Kfz abmelden)' => { 'RUN' => 0, 'ID' => 325881 },
            'Führerschein: Ausstellung – Internationaler Führerschein' => { 'RUN' => 0, 'ID' => 121591 },
            'Neuerteilung der Fahrerlaubnis' => { 'RUN' => 0, 'ID' => 121637 },
            'Niederlassungserlaubnis oder Erlaubnis zum Daueraufenthalt-EU in einen neuen Pass übertragen' => { 'RUN' => 0, 'ID' => 324280 },
            'Fahrerlaubnis zur Fahrgastbeförderung (P-Schein) - Erteilung' => { 'RUN' => 0, 'ID' => 121622 },
    }
);


# # INIT
&init();
# &getRegister();
&getRegisterControls();
# db_createRequestDienstleister(20);
# &getStandorte();
# &getDienstleistungen();
die;

# # MAIN
# &getStandorte() if $config{'RUN_VENUES'};
# &getDienstleistungen() if $config{'RUN_SERVICES'};
# # &run();
# &db_request();
# &Summary();



&programLoop();

# SUBS

sub responseHandlerMech {
    &Delimiter((caller(0))[3]);
    my ($url, $switch) = @_;
    $switch = '' if !$switch;
    
    print "GETTING $url... ";
    
    $mech->get($url);
    say $mech->status();
    
    # my $tree = HTML::TreeBuilder::XPath->new_from_content($mech->content());
    my $tree = HTML::TreeBuilder::XPath->new_from_content(encode('utf-8', $mech->content()));
    my ($success, $code) = &checkSuccess($tree, $mech); # returns 1 or 0 for $success and error/success code for $code
    if (!$success && !$code) {
        &dumpToFile('./output/successcodeundefined', $mech->content);
        die '$success and $code undefined';
    }
    say "success: $success\tcode: $code";
    
    if ($mech->success()) {
        if ($code == -6) {
           say 'Mech->code 200 and success_checkSuccess ' . $code . ' : Starting new appointment search...';
            my $response = &restartHandler($tree);
            return $response;
        }
        
        if ($switch eq 'getRegister') {
            return { 'success' => 1, 'tree' => $tree, 'status' => $mech->status, 'success_checkSuccess' => $success, 'mech' => $mech, 'code_checkSuccess' => $code };
        }
        elsif ($switch eq 'getDienstleistungen' || $switch eq 'filterDienstleistungen' || $switch eq 'getStandorte' || $switch eq 'getTerminDay' || $switch eq 'getRestart' || $switch eq 'getTerminTime' || $switch eq 'captchaHandler') {
            return { 'success' => 1, 'tree' => $tree, 'status' => $mech->status, 'mech' => $mech, 'success_checkSuccess' => $success, 'code_checkSuccess' => $code };
        }
        else {
            return { 'success' => 1, 'tree' => $tree, 'status' => $mech->status, 'mech' => $mech, 'success_checkSuccess' => $success, 'code_checkSuccess' => $code };
        }
    }
    else {
        if ($mech->status() == 404) {
            if ($switch eq 'getRegister') {
                return { 'success' => 1, 'tree' => $tree, 'status' => $mech->status, 'mech' => $mech, 'success_checkSuccess' => $success, 'code_checkSuccess' => $code };
            }
            else {
                say $mech->content();
                die 'No handler';
            }
        }
        elsif ($mech->status() == 429) {
            if ($switch eq 'getDiensleistungen' || $switch eq 'filterDienstleistungen' || $switch eq 'getStandorte' || $switch eq 'getTerminDay' || $switch eq 'getTerminTime') {
                &dumpToFile('./output/429', $mech->content);
                my $treeRateLimit = HTML::TreeBuilder::XPath->new_from_content(encode('utf-8', $mech->content()));
                my $successRate = &rateLimitHandler($treeRateLimit);
                return { 'success' => 0, 'status' => $mech->status, 'success_rateLimitHandler' => $successRate, 'mech' => $mech, 'success_checkSuccess' => $success, 'code_checkSuccess' => $code };
            }
        }
        elsif ($mech->status() == 428) {
            # if ($switch eq 'getTerminTime' || $switch eq 'captchaHandler') {
            #     # say $mech->content();
            #     return { 'success' => 0, 'status' => $mech->status, 'tree' => $tree, 'mech' => $mech, 'success_checkSuccess' => $success, 'code_checkSuccess' => $code };
            # }
            # say $mech->content();
            if ($code == -5 || $code == -7) {
               say 'Mech->code 428 and success_checkSuccess ' . $code . ' : Renewing captcha...';
                my $response = &captchaHandler($tree);
                return $response;
            }
            
        }
        else {
            return { 'success' => 0, 'status' => $mech->status, 'tree' => $tree, 'mech' => $mech, 'success_checkSuccess' => $success, 'code_checkSuccess' => $code };
        }
    }
}

sub restartHandler {
    &Delimiter((caller(0))[3]);
    my $tree = shift;
    
    my $url = '//div[contains(concat(" ", normalize-space(@class), " "), " alert alert-error noprint textile ")]/ul/li[2]/a/@href';
    my $response = &responseHandlerMech($url);
    if ($response->{'success'}) {
        say 'New appointment search successfully started.';
    }
    elsif (!$response->{'success'}) {
        say 'New appointment search not successfull.';
        print Dumper $response;
    }
    return $response;
}

sub getRegister {
    &Delimiter((caller(0))[3]);
    my ($dataset_ref, $user_id) = @_;
    
    my $href = $$dataset_ref{'HREF'};
    
    # my $response = &responseHandlerMech('https://service.berlin.de/terminvereinbarung/termin/time/1576747440/109/', 'getRegister');
    my $response = &responseHandlerMech('https://zinomedia.de/dl/customHTML2.html', 'getRegister');
    
    if ($response->{'success_checkSuccess'}) {
        &getRegisterControls($mech);
        # &submitRegister($mech); 
    }
    else {
        die;
    }
}

sub getRegisterControls {
    &Delimiter((caller(0))[3]);
    use Digest::MD5 qw(md5 md5_hex);
    use Data::Structure::Util qw( unbless );
    my %controls;
    
    for my $service (sort keys %{$config{'SERVICES'}}) {
        my $service_id = $config{'SERVICES'}{$service}{'ID'};
        say "\nSTART GETTING CONTROLS FOR SERVICE '$service'...";
        my $href = "https://service.berlin.de/dienstleistung/$service_id/";
        
        my $response = &responseHandlerMech($href);
        if ($response->{'success'}) {
            say 'success';
            
            my @nodes = $response->{'tree'}->findnodes('//div[@class="behoerdenitem"]/div[@class="row"]');
            say $#nodes;
            # print Dumper $nodes;
            
            foreach my $node (@nodes) {
                if ($node->exists('./div[@class="span5" and (strong)]') || !$node->exists('./div[@class="span2"]/p/strong/a/@href')) {
                    next;
                }
                my $venue = $node->findvalue('./div[@class="span5"]');
                my $terminvereinbarung_url = $node->findvalue('./div[@class="span2"]/p/strong/a/@href');
                my $last = 0;
                
                say "$venue => $terminvereinbarung_url";
                
                my $success = &getTerminDay($terminvereinbarung_url, $service_id, 'getRegisterControls');
                if ($success) {
                    say 'We have appointments.';
                    # print Dumper $data{'SERVICES'}{$service_id}{'AVAILABLE'};
                    
                    foreach my $epochtime (keys %{$data{'SERVICES'}{$service_id}{'AVAILABLE'}}) {
                        foreach my $datetime (keys %{$data{'SERVICES'}{$service_id}{'AVAILABLE'}{$epochtime}{'DATETIMES'}}) {
                            my $register_url = $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$epochtime}{'DATETIMES'}{$datetime}{'HREF'};
                            say $datetime;
                            say $register_url;
                            
                            
                            my $response = &responseHandlerMech($register_url, 'getRegister');
                            if ($response->{'success_checkSuccess'}) {
                                my $form = $mech->form_number(2);
                                my @inputs = $mech->find_all_inputs(
                                    # type       => 'textarea',
                                    # name_regex => qr/^customer/,
                                );
                            
                                my $url = $mech->uri()->as_string;
                                
                                foreach my $i (0 .. $#inputs) {
                                    my $digest;
                                    my $contains_pflichtfeld = 0;
                                    foreach my $key (sort keys %{$inputs[$i]}) {
                                        # print "KEY $key VALUE ";
                                        # say $inputs[0]{$key};
                                        $digest .= "$key$inputs[$i]{$key}";
                                        $contains_pflichtfeld = 1 if $inputs[$i]{$key} =~ m/\*$/;
                                    }
                                    $digest = md5_hex($digest);
                                    if (!$controls{$digest}) {
                                        # $controls{$digest}{'HASH'} = unbless(\%{$inputs[$i]});
                                        $controls{$digest}{'HASH'} = \%{$inputs[$i]};
                                        $controls{$digest}{'FIRST_LVL_PFLICHTFELD'} = $contains_pflichtfeld;
                                        $controls{$digest}{'URI'}{$url} = 1;
                                        push(@{$controls{$digest}{'VENUES'}}, "$venue => $service");
                                    }
                                    else {
                                        push(@{$controls{$digest}{'VENUES'}}, "$venue => $service");
                                    }
                                    
                                }
                                
                                say 'Aborting now...';
                                my $response = &responseHandlerMech('https://service.berlin.de/terminvereinbarung/termin/abort/');
                                sleep(5);
                                $last = 1;
                                last if $last;
                                # say 'Clicking abort now...';
                                # $mech->click_button(id => 'register_abort');
                                # sleep(10);
                            }
                            
                            # print Dumper \%controls;
                            # die;
                            # last;
                            last if $last;
                        }
                        last if $last;
                    }
                    
                }
                
                # print Dumper \%controls;
                # die;
            }
            
            
            die;
        }
        
        
        # &getTerminDay($services{$service_id}{'HREF'}, $service_id);
        # &db_sync($service_id);
    }
    
    die;
    
    
    

    
    # say $digest;
    # print Dumper \@inputs;

}

sub checkSuccess {
    &Delimiter((caller(0))[3]);
    my ($tree, $mech) = @_;
    
    my $xpath_alert = '//div[contains(concat(" ", normalize-space(@class), " "), " alert")]/p[1]';
    my $xpath_alert2 = '//div[contains(concat(" ", normalize-space(@class), " "), " alert")]/div/p';
    my $xpath_alert3 = '//div[@class="zms"]/div[@class="html5-header header"]/h1[@class="title"]';
    
    my $xpath_success = '//div[@class="submit-success-message"]';
    
    if ($tree->exists($xpath_alert)) {
        my $alert = decode('utf-8', trim($tree->findvalue($xpath_alert)));
        say "Alert: $alert";
       
        if ($alert eq 'Sie haben schon einen Termin reserviert. Soll diese Reservierung gelöscht werden oder möchten Sie mit der vorhandenen Reservierung fortfahren?') {
            say 'Code -1: Another reservation is already in progress.';
            &dumpToFile('./output/-1', $mech->content);
            return (0, -1);
        }
        elsif ($alert eq 'Zu ihrer Suche konnten keine Daten ermittelt werden. Dies kann unterschiedliche Gründe haben:') {
            say 'Code -2: Cookie seems outdated';
            &dumpToFile('./output/-2', $mech->content);
            return (0, -2);
        }
        elsif ($alert eq 'Sie haben Ihre Terminvereinbarung abgeschlossen. Sie können nun eine neue Terminssuche starten!') {
            say 'Code -3: Reservation already completed';
            &dumpToFile('./output/-3', $mech->content);
            return (0, -3);
        }
        elsif ($alert eq 'Ihr Termin ist eingetragen worden. Bitte drucken Sie diese Seite aus, damit Sie alle Informationen zur Hand haben. Wichtig für Ihren Besuch ist die Vorlage Ihrer Vorgangsnummer.') {
            say 'Code 1: submitRegister successfull.';
            &dumpToFile('./output/1', $mech->content);
            say trim($tree->findvalue($xpath_success));
            say $tree->findvalue('//span[@class="summary_authKey"]');
            say $tree->findvalue('//span[contains(concat(" ", normalize-space(@class), " "), " summary_processId ")]');
            return (1, 1);
        }
        elsif ($alert eq 'Der eingegebene Text stimmte nicht mit dem Bild überein. Versuchen Sie es noch einmal.') {
            say 'Code -5: Text / Picture mismatch';
            &dumpToFile('./output/-5', $mech->content);
            return (0, -5);
        }
        elsif ($alert eq 'Ihre Auswahl von Standort und Diensteistung hat sich geändert. Möchten Sie mit Ihrer vorherigen Suche fortfahren oder eine neue Terminsuche starten?') {
            say 'Code -6: Location and venue changed. New search or continue.';
            &dumpToFile('./output/-6', $mech->content);
            return (0, -6);
        }
        else {
            say $mech->content();
            say 'NO XPATH SPECIFIED DETECTED';
            &dumpToFile('./output/noxpathspecified', $mech->content);
            # my $string = trim($tree->findvalue($xpath_alert));
            say $alert;
            # $string = decode('utf-8', $string);
            # say $string;
            # say utf8::is_utf8($string); # since Perl 5.8.1
            # say utf8::valid($string);
            die;
        }

    }
    elsif ($tree->exists($xpath_alert2)) {
        my $alert = decode('utf-8', trim($tree->findvalue($xpath_alert2)));
        say "Alert2: $alert";
        
        if ($alert eq 'Leider konnte der ausgewählte Termin nicht reserviert werden. Da war jemand schneller als Sie. Bitte versuchen Sie es erneut.') {
            say 'Code -4: Appointment not available anymore and booked by another person.';
            &dumpToFile('./output/-4', $mech->content);
            return (0, -4);
        }
    }
    elsif ($tree->exists($xpath_alert3)) {
        my $alert = decode('utf-8', trim($tree->findvalue($xpath_alert3)));
        say "Alert 3: $alert";
        
        if ($alert eq 'Bitte verifizieren sie sich') {
            say 'Code -7: Verification needed.';
            &dumpToFile('./output/-7', $mech->content);
            return (0, -7);
        }
        elsif ($alert eq 'Terminvereinbarung') {
            say 'Code 3: Normal Terminvereinbarung';
            &dumpToFile('./output/-7', $mech->content);
            return (1, 3);
        }
    }
    else {
       say 'Code 2: NO XPATH SPECIFIED DETECTED. Everything fine?';
       &dumpToFile('./output/2', $mech->content);
       return (1, 2);
    }
   
}

sub submitRegister {
    &Delimiter((caller(0))[3]);
    my $mech = shift;
    
    
    # print Dumper $mech->all_form_with_fields( ['familyName'] );
    my $form = $mech->form_number(2);
    # print Dumper $result;
    $mech->set_fields( 'familyName' => 'John Smith', 'email' => 'shoty.at.cs@gmx.net', );
    $mech->tick('agbgelesen', 1);
    $mech->click_button(id => 'register_submit');
    
    say $mech->status();
    if ($mech->success()) {
        say $mech->content;
        &dumpToFile('./output/success', $mech->content);
        my $tree_success = HTML::TreeBuilder::XPath->new_from_content($mech->content());
        &checkSuccess($tree_success);
    }
    else {
        if ($mech->status() == 404) {
            say $mech->content();
            say 'PAGE NOT FOUND';
        }
        die;
    }
}

sub programLoop {
    &Delimiter((caller(0))[3]);
    
    for (;;) {
        # INIT
        &init();
        
        # MAIN
        &getStandorte() if $config{'RUN_VENUES'};
        &getDienstleistungen() if $config{'RUN_SERVICES'};
        &db_request();
        &Summary();
        
        &sleepCountdown($config{'SLEEP_LOOP_MIN'} * 60);
    }
}

sub db_createRequestDienstleister {
    &Delimiter((caller(0))[3]);
    my $request_id = shift;
    my $values;
    
    my $string = &db_query("Select terminsnipe.services.HREF From terminsnipe.request Inner Join terminsnipe.services On terminsnipe.services.ID = terminsnipe.request.SERVICE_ID Where terminsnipe.request.ID = $request_id", $connection, 'fetchrow');
    my @dienstleister = getDienstleisterlist($string);
    
    foreach my $id (@dienstleister) {
        $values .= "('$request_id', '$id'), ";
    }
    my $sql = substr("INSERT INTO `request_dienstleister` (`REQUEST_ID`, `VENUE_ID`) VALUES $values", 0, -2);
    # say $sql;
    
    &db_query("DELETE FROM `request_dienstleister` WHERE `REQUEST_ID` = $request_id", $connection, 'do');
    &db_query($sql, $connection, 'do');
}

sub getDienstleisterlist {
    &Delimiter((caller(0))[3]); 
    my $string = shift;
    
    
    $string =~ m/dienstleisterlist=(.+?)&/;
    return split /,/, $1;
}

sub db_request {
    &Delimiter((caller(0))[3]); 
    my %request;
    
    my $response = &db_query("SELECT `ID`, `USER_ID`, `SERVICE_ID`, `DATETIME_FROM`, `DATETIME_TO`, `TIMESTAMP` FROM `request` WHERE `EXPIRED` = 0;", $connection, 'selectall_arrayref');
    if (@$response) {
        for my $i (0 .. $#{$response}) {
            my $dataset = $$response[$i];
            $request{$$dataset[2]}{$$dataset[0]}{'ID'} = $$dataset[0];
            $request{$$dataset[2]}{$$dataset[0]}{'USER_ID'} = $$dataset[1];
            $request{$$dataset[2]}{$$dataset[0]}{'SERVICE_ID'} = $$dataset[2];
            $request{$$dataset[2]}{$$dataset[0]}{'DATETIME_FROM'} = $$dataset[3];
            $request{$$dataset[2]}{$$dataset[0]}{'DATETIME_TO'} = $$dataset[4];
            $request{$$dataset[2]}{$$dataset[0]}{'TIMESTAMP'} = $$dataset[5];
            $services{$$dataset[2]}{'HREF_GENERATED'} = &db_generateHREF($$dataset[0], $$dataset[2]);
        }
        print Dumper \%request;
        
        foreach my $service_id (keys %request) {
            &startFork($service_id, \%request);
        }
        $pm->wait_all_children;
        say 'All childs are done. Continuing with parent...';
        foreach my $service_id (keys %request) {
            &db_find_appointment($service_id, \%request);
        }
    }
    else {
        say 'No requests in db.';
    }
}

sub db_generateHREF {
    &Delimiter((caller(0))[3]);
    my ($request_id, $service_id) = @_;
    
    my $href_generated = $services{$service_id}{'HREF'};
    my $response = &db_query("SELECT `VENUE_ID` FROM `request_dienstleister` WHERE `REQUEST_ID` = $request_id;", $connection, 'selectall_arrayref');
    
    my $dienstleisterlist;
    for my $i (0 .. $#{$response}) {
        $dienstleisterlist .= "$$response[$i][0],";
    }
    $dienstleisterlist = substr($dienstleisterlist, 0, -1);
    $href_generated =~ m/dienstleisterlist=(.+?)&/;
    $href_generated =~ s/$1/$dienstleisterlist/;
    
    return $href_generated;
}

sub startFork {
    &Delimiter((caller(0))[3]);
    my ($service_id, $request_ref) = @_;
    
    my $pid = $pm->start($service_id) and next;
    
    &run_from_request($service_id);
    &db_sync($service_id);
    # &db_find_appointment($service_id, $request_ref);
 
    $pm->finish;
}

sub run_from_request {
    &Delimiter((caller(0))[3]);
    my $service_id = shift;
    
    my $service = $services{$service_id}{'NAME'};
    my $href = $services{$service_id}{'HREF_GENERATED'};
    
    say "START RUN FOR SERVICE '$service'...";
    &getTerminDay($href, $service_id);
}

sub db_find_appointment {
    &Delimiter((caller(0))[3]);
    my ($service_id, $request_ref) = @_;
    
    foreach my $id (keys %{$$request_ref{$service_id}}) {
        my $query = "Select terminsnipe.datetime.DATETIME, terminsnipe.mapping.SERVICE_ID, terminsnipe.mapping.VENUE_ID, terminsnipe.mapping.HREF As HREF, terminsnipe.venues.NAME As VENUE_NAME, terminsnipe.datetime.EPOCHTIME, terminsnipe.mapping.ID As MAPPING_ID From terminsnipe.datetime Inner Join terminsnipe.mapping On terminsnipe.mapping.DATETIME_ID = terminsnipe.datetime.EPOCHTIME Inner Join terminsnipe.venues On terminsnipe.mapping.VENUE_ID = terminsnipe.venues.ID Where Not terminsnipe.datetime.DATETIME > '$$request_ref{$service_id}{$id}{'DATETIME_TO'}' And Not terminsnipe.datetime.DATETIME < '$$request_ref{$service_id}{$id}{'DATETIME_FROM'}' And terminsnipe.mapping.SERVICE_ID = $service_id;";
        say $query;
        my $response = &db_query($query, $connection, 'selectall_hashref', 'EPOCHTIME');
        print Dumper $response;

        say 'Possible appointments: ' . scalar(keys(%$response));
        if (%$response) {
            say 'defined';
            
            &db_insertAppointment($response, $$request_ref{$service_id}{$id});
        }
    }
}

sub db_insertAppointment {
    &Delimiter((caller(0))[3]);
    my ($response, $dataset_ref) = @_;
    my $user_id = $$dataset_ref{'USER_ID'};
    my $request_id = $$dataset_ref{'ID'};
    
    foreach my $epochtime (keys %{$response}) {
        if (!$$response{$epochtime}{'RESERVED'}) {
            say "Reserving appointment $$response{$epochtime}{MAPPING_ID} for USER_ID $user_id now...";
            
            &getRegister($$response{$epochtime}, $user_id);
            
            &db_query("INSERT INTO `appointments` (`MAPPING_ID`, `USER_ID`) VALUES ('$$response{$epochtime}{MAPPING_ID}', '$user_id');", $connection, 'do');
            $$response{$epochtime}{'RESERVED'} = 1;
            
            say 'Deleting request with ID $request_id from request and request_dienstleister...';
            &db_query("DELETE FROM `request` WHERE `ID` = $request_id;", $connection, 'do');
            &db_query("DELETE FROM `request_dienstleister` WHERE `REQUEST_ID` = $request_id;", $connection, 'do');
            last;
        }
        else {
            say "Appointment $epochtime is available but already reserved in loop by another user.";
        }
    }
}

sub db_query {
    # &Delimiter((caller(0))[3]); 
    my ($query, $connection, $switch, @rest) = @_;
    my $response;
    
    # say 'DATABASE CONNECTION STATE: ' . $connection->ping;
    
    # $query = decode("utf-8", $query);
    # $query = encode( "iso-8859-1", $query );
    
    my $statement;
    if ($switch eq 'fetchrow') {
        $statement = $connection->prepare($query);
        $statement->execute();
        $response = $statement->fetchrow();
    }
    elsif ($switch eq 'fetchall_hashref') {
        $statement = $connection->prepare($query);
        $statement->execute();
        $response = $statement->fetchall_hashref($rest[0]);
    }
    elsif ($switch eq 'fetchall_arrayref') {
        $statement = $connection->prepare($query);
        $statement->execute();
        $response = $statement->fetchall_arrayref();
    }
    elsif ($switch eq 'do') {
        $response = $connection->do($query) or die $connection->errstr;
    }
    elsif ($switch eq 'selectall_hashref') {
        $response = $connection->selectall_hashref($query, $rest[0]) or warn $connection->errstr;
    }
    elsif ($switch eq 'selectall_array') {
        my @response = $connection->selectall_array($query) or die $connection->errstr;
        return \@response;
    }
    elsif ($switch eq 'selectall_arrayref') {
        $response = $connection->selectall_arrayref($query) or die $connection->errstr;
    }
    elsif ($switch eq 'selectrow_hashref') {
        $response = $connection->selectrow_hashref($query) or die $connection->errstr;
    }
    
    $statement->finish if $statement;
    return $response;
}

sub connectToMySql {
    &Delimiter((caller(0))[3]); 
    my ($db) = @_;
    
    # assign the values to your connection variable
    my $connectionInfo = "dbi:mysql:$db;$config{'MYSQL'}{'ACCESS'}{'hostname'}";
    
    # make connection to database
    # my $l_connection = DBI->connect($connectionInfo, $config{'MYSQL'}{'ACCESS'}{'user'}, $config{'MYSQL'}{'ACCESS'}{'pw'}, $config{'MYSQL'}{'OPTIONS'}) || warn "MySQL conncet failed: $DBI::errstr";
    # return $l_connection;
    
    my $l_connection = DBIx::Connector->new($connectionInfo, $config{'MYSQL'}{'ACCESS'}{'user'}, $config{'MYSQL'}{'ACCESS'}{'pw'}, {
        RaiseError => 1,
        AutoCommit => 1,
    });
    return $l_connection->dbh;
}

sub run {
    &Delimiter((caller(0))[3]);   
    
    for my $service (keys %{$config{'SERVICES'}}) {
        if ($config{'SERVICES'}{$service}{'RUN'}) {
            my $service_id = $config{'SERVICES'}{$service}{'ID'};
            say "START RUN FOR SERVICE '$service'...";
            &getTerminDay($services{$service_id}{'HREF'}, $service_id);
            &db_sync($service_id);
        }
        else {
            # say "DISABLED $service";
        }
    }
}

sub db_sync {
    &Delimiter((caller(0))[3]);
    my $service_id = shift;
    
    &db_query("DELETE FROM `mapping` WHERE `SERVICE_ID` = '$service_id';", $connection, 'do');
    
    foreach my $day (keys %{$data{'SERVICES'}{$service_id}{'AVAILABLE'}}) {
        foreach my $datetime (keys %{$data{'SERVICES'}{$service_id}{'AVAILABLE'}{$day}{'DATETIMES'}}) {
            my $epochtime = $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$day}{'DATETIMES'}{$datetime}{'EPOCHTIME'};
            my $venue_id = $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$day}{'DATETIMES'}{$datetime}{'VENUE_ID'};
            my $href = $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$day}{'DATETIMES'}{$datetime}{'HREF'};
            
            say "$service_id\t$venue_id\t$epochtime";
            
            # say "INSERT INTO `datetime` (`EPOCHTIME`, `DATETIME`) VALUES ('$data{'SERVICES'}{$service_id}{'AVAILABLE'}{$day}{'DATETIMES'}{$datetime}{'EPOCHTIME'}', '$datetime') ON DUPLICATE KEY UPDATE `DATETIME` = `DATETIME`;";
            # say "INSERT INTO `datetime` (`EPOCHTIME`, `DATETIME`) SELECT $epochtime, '$datetime' FROM dual WHERE NOT EXISTS (SELECT 1 FROM `datetime` WHERE `EPOCHTIME`= $epochtime);";
            # say "INSERT INTO `mapping` (`SERVICE_ID`, `VENUE_ID`, `DATETIME_ID`, `HREF`) SELECT '$service_id', '$venue_id', '$epochtime', '$href' FROM dual WHERE NOT EXISTS (SELECT 1 FROM `mapping` WHERE `SERVICE_ID` = '$service_id' AND `VENUE_ID` = '$venue_id' AND `DATETIME_ID` = '$epochtime');";
            
            # &db_query("INSERT INTO `datetime` (`EPOCHTIME`, `DATETIME`) VALUES ('$data{'SERVICES'}{$service_id}{'AVAILABLE'}{$day}{'DATETIMES'}{$datetime}{'EPOCHTIME'}', '$datetime') ON DUPLICATE KEY UPDATE `DATETIME` = `DATETIME`;", $connection, 'do');
            &db_query("INSERT INTO `datetime` (`EPOCHTIME`, `DATETIME`) SELECT $epochtime, '$datetime' FROM dual WHERE NOT EXISTS (SELECT 1 FROM `datetime` WHERE `EPOCHTIME`= $epochtime);", $connection, 'do');
            &db_query("INSERT INTO `mapping` (`SERVICE_ID`, `VENUE_ID`, `DATETIME_ID`, `HREF`) SELECT '$service_id', '$venue_id', '$epochtime', '$href' FROM dual WHERE NOT EXISTS (SELECT 1 FROM `mapping` WHERE `SERVICE_ID` = '$service_id' AND `VENUE_ID` = '$venue_id' AND `DATETIME_ID` = '$epochtime');", $connection, 'do');
        }
    }
    
}

sub createMysqlDateTime {
    # &Delimiter((caller(0))[3]);
    my $string = shift;
    
    my $dt = $strp->parse_datetime($string);
    return DateTime::Format::MySQL->format_datetime($dt);    
}

sub getDienstleistungen {
    &Delimiter((caller(0))[3]);
    
    my $response = &responseHandlerMech($config{'URL_DIENSTLEISTUNGEN'}, 'getDienstleistungen');
    
    if ($response->{'success'}) {
        &parseDienstleistungen($response->{'tree'});
    }
    elsif (!$response->{'success'} && $response->{'success_rateLimitHandler'}) {
        say 'No success in getDienstleistungen and rateLimitHandler successfull. Retrying...';
        &getDienstleistungen();
    }
    elsif (!$response->{'success'} && !$response->{'success_rateLimitHandler'}) {
        die 'No success in getDienstleistungen and rateLimitHandler unsuccessfull.';
    }
    
    if ($config{'FILTER_SERVICES_TO_OUTPUT'} || $config{'DB_REFRESH_SERVICES'}) {
        &filterDienstleistungen();
        &db_insertServicesVenues('services') if $config{'DB_REFRESH_SERVICES'};
    }
    
    if ($config{'DUMP_SERVICES_TO_FILE'}) {
        &dumpToFile($config{'SERVICES_PATH'}, \%services);
    }
}

sub db_insertServicesVenues {
    &Delimiter((caller(0))[3]);
    my $switch = shift;
    my ($database, $hash_ref);
    
    if ($switch eq 'services') {
        $database = 'services';
        $hash_ref = \%services;
    }
    elsif ($switch eq 'venues') {
        $database = 'venues';
        $hash_ref = \%venues;
    }
    
    my $response = &db_query("delete FROM $database", $connection, 'do');
    
    foreach my $id (sort keys %{$hash_ref}) {
        my $query = "INSERT INTO `$database` (`ID`, `NAME`, `HREF`) VALUES ('$$hash_ref{$id}{'ID'}', '$$hash_ref{$id}{'NAME'}', '$$hash_ref{$id}{'HREF'}');";
        my $response = &db_query($query, $connection, 'do');
    }
}

sub filterDienstleistungen {
    &Delimiter((caller(0))[3]);
    
    # DATA DUMPER UTF8 HACK
    local *Data::Dumper::qquote = sub { qq["${\(shift)}"] };
    local $Data::Dumper::Useperl = 1;
    
    open my $FILE, '>>:encoding(UTF-8)', $config{'FILTER_SERVICES_PATH'};
    foreach my $id (keys %services) {
        
        my $response = &responseHandlerMech($services{$id}{'HREF'}, 'filterDienstleistungen');
        
        if ($response->{'success'}) {
            my $tree = $response->{'tree'};
            my $name = $services{$id}{'NAME'};
                
            print "\t$name => ";
            
            if (!$tree->exists('//div[contains(concat(" ", normalize-space(@class), " "), " zmstermin-multi inner ")]')) {
                say '0';
                delete $services{$id};
            }
            else {
                my $href = $tree->findvalue('//div[contains(concat(" ", normalize-space(@class), " "), " zmstermin-multi inner ")]/a/@href');
                my $entry = "'$name' => { 'RUN' => 0, 'ID' => $services{$id}{'ID'} },";
                print '1 => ' . $tree->findvalue('//div[contains(concat(" ", normalize-space(@class), " "), " zmstermin-multi inner ")]/a') . " | $entry\n";
                say $FILE $entry if $config{'FILTER_SERVICES_TO_OUTPUT'};
                $services{$id}{'HREF'} = $href;
            }
        }
        elsif (!$response->{'success'} && $response->{'success_rateLimitHandler'}) {
            say 'No success in filterDienstleistungen and rateLimitHandler successfull. Retry here?';
            die;
        }
        elsif (!$response->{'success'} && !$response->{'success_rateLimitHandler'}) {
            die 'No success in filterDienstleistungen and rateLimitHandler unsuccessfull.';
        }
    }
    close $FILE;
}

sub parseDienstleistungen {
    &Delimiter((caller(0))[3]);
    my $tree = shift;
    
    foreach my $dataset ($tree->findnodes('//li[contains(concat(" ", normalize-space(@class), " "), " topic-dls row-fluid ")]')) {
        my $id = basename($dataset->findvalue('./a/@href'));
        my $href = $dataset->findvalue('./a/@href');
        my $name = trim($dataset->findvalue('./a'));
        
        $services{$id}{'ID'} = $id;
        $services{$id}{'HREF'} = "$config{'URL_ROOT'}$href";
        $services{$id}{'NAME'} = $name;
    }
}

sub getStandorte {
    &Delimiter((caller(0))[3]);
    
    my $response = &responseHandlerMech($config{'URL_STANDORTE'}, 'getStandorte');
    
    if ($response->{'success'}) {
        &parseStandorte($response->{'tree'});
        &db_insertServicesVenues('venues') if $config{'DB_REFRESH_VENUES'};
        dumpToFile($config{'VENUES_PATH'}, \%venues) if $config{'VENUES_TO_FILE'};
    }
    elsif (!$response->{'success'} && $response->{'success_rateLimitHandler'}) {
        say 'No success in getStandorte and rateLimitHandler successfull. Retry?';
        die;
    }
    elsif (!$response->{'success'} && !$response->{'success_rateLimitHandler'}) {
        die 'No success in getStandorte and rateLimitHandler unsuccessfull.';
    }
}

sub dumpToFile {
    &Delimiter((caller(0))[3]);
    my ($filepath, $ref) = @_;
    
    # DATA DUMPER UTF8 HACK
    # no warnings 'redefine';
    local *Data::Dumper::qquote = sub { qq["${\(shift)}"] };
    local $Data::Dumper::Useperl = 1;
    
    open my $FILE, '>:encoding(UTF-8)', $filepath;
    print $FILE Dumper $ref;
    close $FILE;
}

sub parseStandorte {
    &Delimiter((caller(0))[3]);
    my $tree = shift;
    
    foreach my $dataset ($tree->findnodes('//li[contains(concat(" ", normalize-space(@class), " "), " topic-dls ")]')) {
        my $id = basename($dataset->findvalue('./a/@href'));
        my $href = $dataset->findvalue('./a/@href');
        my $name = trim($dataset->findvalue('./a'));
        
        say $name;
        
        $venues{$id}{'ID'} = $id;
        $venues{$id}{'HREF'} = "$config{'URL_ROOT'}$href";
        $venues{$id}{'NAME'} = $name;
    }
}

sub init {
    &Delimiter((caller(0))[3]);
    
    ($zsmappointment, $ts, $mech, $ua, $cookie_jar, $strp, $connection, $strp_datetime, $pm) = (undef) x 9;
    undef %data;
    
    # COOKIES & ARGS
    $cookie_jar = HTTP::Cookies->new(ignore_discard => 1 );
    if (@ARGV) {
        $cookie_jar->set_cookie( 0, 'Zmsappointment', $ARGV[0], '/', 'service.berlin.de', undef, 1, undef, undef, 1, ) if $ARGV[0];
        $cookie_jar->set_cookie( 0, 'TS01fa49ef', $ARGV[1], '/', 'service.berlin.de', undef, 1, undef, undef, 1, ) if $ARGV[1];
    }
    elsif (!@ARGV && -e $config{'COOKIE_PATH'}) {
        say "Loading $config{'COOKIE_PATH'}...";
        $cookie_jar->load($config{'COOKIE_PATH'});
    }
    
    # STRP DATETIME
    $strp = DateTime::Format::Strptime->new(
        pattern   => '%d %b %Y %H:%M',
        locale => 'de_DE',
    );
    
    $strp_datetime = DateTime::Format::Strptime->new(
        pattern   => '%Y-%m-%d %T',
        locale => 'de_DE',
    );
    

    # BROWSER
    if ($config{'BROWSER'} eq 'MECH') {
        use WWW::Mechanize ();
        $mech = WWW::Mechanize->new( 
            cookie_jar => $cookie_jar,
            agent => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36',
            autocheck => 0,
        );
    }
    elsif ($config{'BROWSER'} eq 'LWP') {
        use LWP::UserAgent;
        $ua = LWP::UserAgent->new(
            ssl_opts => {
                    SSL_version => 'TLSv12:!SSLv2:!SSLv3:!TLSv1:!TLSv11',
            },
            cookie_jar => $cookie_jar,
            agent => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36',
        );
    }
    else {
        die "$config{'BROWSER'} not defined.";
    }
    
    # MYSQL
    $connection = &connectToMySql($config{'MYSQL'}{'ACCESS'}{'database'});
    
    # SERVICE AND VENUE HASH
    $connection->do('set names utf8');
    %venues = %{&db_query("SELECT `ID`, `NAME`, `HREF` FROM `venues`", $connection, 'fetchall_hashref', 'ID')};
    %services = %{&db_query("SELECT `ID`, `NAME`, `HREF` FROM `services`", $connection, 'fetchall_hashref', 'ID')};
    
    # EMPTY DIRECTORIES
    # unlink glob "'$config{'CAPTCHA_DIR'}/*'";
    unlink glob "'$config{'OUTPUT_DIR'}/*'";
    
    # CLEAN OLD DATETIMES, FLAG EXPIRED REQUESTS
    &db_query("DELETE FROM `datetime` WHERE `DATETIME` < NOW();", $connection, 'do');
    &db_query("UPDATE request SET `EXPIRED` = 1 WHERE `DATETIME_TO` < NOW();", $connection, 'do');
    
    # FORK MANAGER
    $pm = Parallel::ForkManager->new(scalar(keys(%services)));
    # $pm = Parallel::ForkManager->new(1);
    $pm->run_on_start(
        sub {
            my ($pid, $ident)=@_;
            say "\n** $ident started, pid: $pid";
    });
    $pm->run_on_finish(
        sub {
            my ($pid, $exit_code, $ident) = @_;
            say "\n** $ident just got out of the pool with PID $pid and exit code: $exit_code";
    });
    $pm->run_on_wait(
        sub {
            # my ($pid, $ident) = @_;
            say "\n** a child is waiting now...";
    });
}

sub Summary {
    &Delimiter((caller(0))[3]);
    
    &dumpToFile($config{'OUTPUT_PATH'}, \%data);
    $connection->disconnect();
}

sub getTerminDay {
    &Delimiter((caller(0))[3]);
    my ($url, $service_id, $switch) = @_;
    
    my $response = &responseHandlerMech($url, 'getTerminDay');
    
    if ($response->{'success'}) {
        my $success = &parseTerminDay($response->{'tree'}, $service_id, $switch);
        
        if ($switch eq 'getRegisterControls' && $success) {
            say 'No need to go next. Returning...';
            return 1;
        }
        elsif (!$success) {
            say 'No success in getTerminDay';
            # say $response->{'mech'}->content();
            # die;
        }
            
        if ($response->{'tree'}->exists('//th[contains(concat(" ", normalize-space(@class), " "), " next ")]/a/@href')) {
            say 'NEXT EXISTS';
            if ($config{'FOLLOW_NEXT'}) {
                my $epochtime = basename($response->{'tree'}->findvalue('//th[contains(concat(" ", normalize-space(@class), " "), " next ")]/a/@href'));
                my $url = "$config{'URL_TERMIN_DAY'}/$epochtime";
                &getTerminDay($url, $service_id, $switch);
            }
        }
    }
    elsif (!$response->{'success'} && $response->{'success_rateLimitHandler'}) {
        say 'No success in getTerminDay and rateLimitHandler successfull. Retry?';
        die;
    }
    elsif (!$response->{'success'} && !$response->{'success_rateLimitHandler'}) {
        die 'No success in getTerminDay and rateLimitHandler unsuccessfull.';
    }
}

sub parseTerminDay {
    &Delimiter((caller(0))[3]);
    my ($tree, $service_id, $switch) = @_;
    
    my $xpath = '//div[contains(concat(" ", normalize-space(@class), " "), " calendar-month-table ")]';
    my $xpath_restart = '//div[contains(concat(" ", normalize-space(@class), " "), " alert alert-error noprint textile ")]/ul/li[2]/a/@href';
    if ($tree->exists($xpath)) {
        foreach my $table ($tree->findnodes($xpath)) {
            my ($month, $year) = split(/ /, $table->findvalue('./table/thead/tr/th[contains(concat(" ", normalize-space(@class), " "), " month ")]'));
            foreach my $buchbar ($table->findnodes('./table/tbody/tr/td[contains(concat(" ", normalize-space(@class), " "), " buchbar ")]')) {
                my $id = basename($buchbar->findvalue('./a/@href'));
                
                if ($data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}) {
                    say 'ALREADY PARSED';
                    next;
                }
                else {
                    my $day = $buchbar->findvalue('./a');
                    $data{'SERVICES'}{$service_id}{'SERVICE_ID'} = $service_id;
                    $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'MONTH'} = $month;
                    $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'YEAR'} = $year;
                    $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'DAY'} = $day;
                    $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'ID'} = $id;
                    $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'DATETIME'} = &createMysqlDateTime("$data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'DAY'} $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'MONTH'} $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'YEAR'} 00:00");
                    
                    my $url = "$config{'URL_TERMIN_TIME'}/$id";
                    &getTerminTime($url, $id, $service_id);
                    
                    if ($switch eq 'getRegisterControls') {
                        if (scalar keys %{$data{'SERVICES'}{$service_id}{'AVAILABLE'}} > 0) {
                            say 'Successfully parsed at least 1 appointment. Returning...';
                            return 1;
                        }
                    }
                }
            }
        }
        # return 1;
    }
    elsif ($tree->exists($xpath_restart)) {
        say 'XPATH ALERT DETECTED';
        my $url = $config{'URL_ROOT'} . $tree->findvalue($xpath_restart);
        say $url;
        
        my $response = &responseHandlerMech($url, 'getRestart');
        if ($response->{'success'}) {
            &parseTerminDay($response->{'tree'}, $service_id, $switch);
        }
    }
    else {
        say 'Parsing error.';
        return 0;
    }
}

sub getTerminTime {
    &Delimiter((caller(0))[3]);
    my ($url, $id, $service_id, $switch) = @_;
    
    my $response = &responseHandlerMech($url, 'getTerminTime');
    if ($response->{'success'}) {
        my $success = &parseTerminTime($response->{'tree'}, $id, $service_id);
        if (!$success) {
            say $mech->content();
            say 'No success in parseTerminTime. Retrying...';
            &getTerminTime($url, $id, $service_id); # RETRY
        }
    }
    # elsif (!$response->{'success'} && $response->{'status'} == 428) {
    #     say 'No success in parseTerminTime. Renewing captcha...';
    #     my $success = &captchaHandler($response->{'tree'});
    #     &getTerminTime($url, $id, $service_id); # RETRY
    # }
    # elsif (!$response->{'success'} && $response->{'status'} == 429) {
    #     say 'No success in getStandorte and rateLimitHandler detected. Retrying...';
    #     &getTerminTime($url, $id, $service_id); # RETRY
    # }
}

sub rateLimitHandler {
    &Delimiter((caller(0))[3]);
    my $tree = shift;
    
    my $error = $tree->findvalue('//div[contains(concat(" ", normalize-space(@class), " "), " offset2 ")]/h1');
    say $error;
    
    if ($error eq 'Zu viele Zugriffe') {
        say 'Rate Limit detected.';
        &sleepCountdown($config{'RATE_LIMIT_SLEEP_MIN'} * 60);
        return 1;
    }
    else {
        say 'Could not resolve error.';
        return 0;
    }
}

sub sleepCountdown {
    &Delimiter((caller(0))[3]);
    my $countdown = shift;
    
    $|++; 
    for (my $i=$countdown; $i >= 0; $i--) {
        printf "Sleeping for %s seconds...\r", $i;
        sleep(1);
    }
    return 1;
}

sub captchaHandler {
    &Delimiter((caller(0))[3]);
    my $tree = shift;
    
    my $captchaBase64 = $tree->findvalue('//fieldset[contains(concat(" ", normalize-space(@class), " "), " well ")]/p/img/@src');
    my $filename = &base64ToFile($captchaBase64);
    # system("cacaview $filename");
    system("tiv -0 $filename");
    
    say 'Please solve captcha ' . basename($filename) . ' to proceed: ';
    my $captcha_text = <STDIN>;
    chomp $captcha_text;
    my $urlCaptcha = "$config{'URL_HUMAN'}/?captcha_text=$captcha_text";
    
    my $response = &responseHandlerMech($urlCaptcha, 'captchaHandler');
    if ($response->{'success'}) {
        say 'Captcha successfully solved. Saving cookie...';
        $cookie_jar->save($config{'COOKIE_PATH'});
    }
    return $response;
    # elsif (!$response->{'success'} && $response->{'status'} == 428) {
    #     # return 0;
    #     return $response;
    # }
}

sub trim {
    my $string = shift;
    $string =~ s/^\s+|\s+$//g;
    return $string;
}

sub parseTerminTime {
    &Delimiter((caller(0))[3]);
    my ($tree, $id, $service_id) = @_;
    
    my $xpath = '//div[contains(concat(" ", normalize-space(@class), " "), " timetable ")]/table/tbody/tr';
    my $xpath_stop = '//link[contains(concat(" ", normalize-space(@rel), " "), " alternate ")]/@href';
    
    if ($tree->exists($xpath)) {
        my $time;
        foreach my $buergeramt_dataset ($tree->findnodes($xpath)) {
            if ($buergeramt_dataset->findvalue('./th[1]')) {
                $time = trim($buergeramt_dataset->findvalue('./th[1]'));
                $time = &createMysqlDateTime("$data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'DAY'} $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'MONTH'} $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'YEAR'} $time");
                # print "Parsing for $time...";
            }
            foreach my $buergeramt ($buergeramt_dataset->findnodes('./td[contains(concat(" ", normalize-space(@class), " "), " frei ")]')) {
                my $name = trim($buergeramt->findvalue('./a'));
                my $href = $buergeramt->findvalue('./a/@href');
                say $time;
                $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'DATETIMES'}{$time}{'VENUE'} = $name;
                $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'DATETIMES'}{$time}{'HREF'} = "$config{'URL_ROOT'}$href";
                
                # say $name . ' needs match: ' . grep{ $venues{$_}{'NAME'} eq $name} keys %venues;
                ($data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'DATETIMES'}{$time}{'VENUE_ID'}) = grep{ $venues{$_}{'NAME'} eq $name} keys %venues;
                $data{'SERVICES'}{$service_id}{'AVAILABLE'}{$id}{'DATETIMES'}{$time}{'EPOCHTIME'} = DateTime::Format::DateParse->parse_datetime($time)->epoch();
            }
        }
        return 1;
    }
    elsif ($tree->findvalue($xpath_stop) eq '/terminvereinbarung/termin/stop/') {
        say "XPATH_STOP DETECTED";
        die;
    }
    else {
        say 'Parsing error.';
        return 0;
    }
}

sub base64ToFile {
    &Delimiter((caller(0))[3]);
    my $base64 = shift;
    
    my $timestamp = &GetTimestamp('YMDHMS');
    $base64 =~ s/data:image\/(.+?)\;base64,//;
    my $decoded= decode_base64($base64);
    my $filename = "$config{'CAPTCHA_DIR'}/$timestamp.png";
    open my $fh, '>', $filename or die $!;
        binmode $fh;
        print $fh $decoded;
    close $fh;
    return $filename;
}

sub Delimiter {
    my $SubName = shift;
    print "\n" . "-" x 80 . "\nSUB " . $SubName . "\n" . '-' x 80 . "\n";
}

sub GetTimestamp {
    #&Delimiter((caller(0))[3]);
	my $switch = shift;
	
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
    
    my $nice_timestamp;
    if ($switch eq 'YMDHMS') {
    	$nice_timestamp = sprintf ( "%04d%02d%02d_%02d%02d%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec);
    }
    elsif ($switch eq 'YMD') {
    	$nice_timestamp = sprintf ( "%04d%02d%02d", $year+1900,$mon+1,$mday);
    }
    elsif ($switch eq 'year') {
    	$nice_timestamp = $year+1900;
    }
    elsif ($switch eq 'month') {
    	$nice_timestamp = $mon+10;
    }
    else {
    	print "Invalid/no switch detected. Use: 'YMDHMS' / 'YMD'\n";
    }
    
    return $nice_timestamp;
}