integrated_xml_ipv6_crawler.php 10.7 KB
Newer Older
floh1111's avatar
floh1111 committed
1
<?php
2
	$starttime = time();
3 4 5 6
	//deny if script is not called by the server directly
	if(!empty($_SERVER['REMOTE_ADDR'])) {
		die("This script can only be run by the server directly.");
	}
boombuler's avatar
boombuler committed
7

8 9 10 11 12
	//set include paths
	if (empty($_SERVER["REQUEST_URI"])) {
		$path = dirname(__FILE__)."/";
		set_include_path(get_include_path() . PATH_SEPARATOR . $path);
	}
boombuler's avatar
boombuler committed
13

14
	//get depencies
Tim Niemeyer's avatar
Tim Niemeyer committed
15
	$crawler = true;
16 17
	require_once('runtime.php');
	require_once(ROOT_DIR.'/lib/core/Routerlist.class.php');
18
	require_once(ROOT_DIR.'/lib/core/RouterStatus.class.php');
19 20 21 22
	require_once(ROOT_DIR.'/lib/core/Networkinterface.class.php');
	require_once(ROOT_DIR.'/lib/core/Iplist.class.php');
	require_once(ROOT_DIR.'/lib/api/crawl.class.php');
	require_once(ROOT_DIR.'/lib/core/ConfigLine.class.php');
23
	require_once(ROOT_DIR.'/lib/core/OriginatorStatus.class.php');
Tim Niemeyer's avatar
Tim Niemeyer committed
24
	require_once(ROOT_DIR.'/lib/extern/pstools.inc');
boombuler's avatar
boombuler committed
25

26 27 28
	//get offset and limit options (script parameters)
	$router_offset = getopt("o:l:")['o'];
	$router_limit = getopt("o:l:")['l'];
boombuler's avatar
boombuler committed
29

30
	// get configuration values
Tim Niemeyer's avatar
Tim Niemeyer committed
31
	$ping_count = 3; // ping a node X times before fetching data
32 33 34
	$ping_timeout = 1; // set the timout for each ping to X s
	$ping_hard_timeout = 3; // set the timout for each ping command to X s
	$crawl_timeout = 10; // timeout after X seconds on fetching crawldata
35
	$network_connection_ipv6_interface = ConfigLine::configByName("network_connection_ipv6_interface"); //use this interface to connect to ipv6 linc local hosts
36 37
	$crawl_interfaces = explode(",", ConfigLine::configByName("crawl_interfaces"));
	//array("br-mesh", "br-client", "floh_fix", "tata_fix"); //use the ip adresses of these interfaces for crawling
boombuler's avatar
boombuler committed
38

39
	$actual_crawl_cycle = Crawling::getActualCrawlCycle()['id'];
boombuler's avatar
boombuler committed
40

41 42 43 44 45
	echo "Crawling routers $router_offset-$router_limit (offset-limit) with the following options:\n";
	echo "	ping_count: $ping_count\n";
	echo "	ping_timeout: $ping_timeout\n";
	echo "	crawl_timeout: $crawl_timeout\n";
	echo "	network_connection_ipv6_interface: $network_connection_ipv6_interface\n";
46
	echo "	crawl_interfaces: "; foreach($crawl_interfaces as $iface) echo $iface." "; echo "\n";
47
	echo "	actual_crawl_cycle: ".$actual_crawl_cycle."\n";
boombuler's avatar
boombuler committed
48

49 50 51 52 53
	//fetch all routers that need to be crawled by a crawler. Respect offset and limit!
	$routerlist = new Routerlist(false, false, "crawler", false, false, false, false, false,
								 (int)$router_offset, (int)$router_limit, "router_id", "asc");
	foreach($routerlist->getRouterlist() as $key=>$router) {
		echo ($key+1).". crawling Router ".$router->getHostname()." (".$router->getRouterId().")\n";
54
		foreach($crawl_interfaces as $name) {
55 56 57 58 59 60
			echo "	Fetching IP-Addresses of interface ".$name."\n";
			$networkinterface = new Networkinterface(false, $router->getRouterId(), $name);
			if($networkinterface->fetch()) {
				$iplist = new Iplist($networkinterface->getNetworkinterfaceId());
				foreach($iplist->getIplist() as $ip) {
					echo "		Working with ".$ip->getIp()."\n";
61
					$xml_array = array();
62 63
					$ping=false;
					$return = array();
boombuler's avatar
boombuler committed
64

65
					if($ip->getNetwork()->getIpv()==6)
Tim Niemeyer's avatar
Tim Niemeyer committed
66
						$command = "ping6 -c $ping_count -w ".($ping_count+1)*$ping_timeout." -W $ping_timeout -I $network_connection_ipv6_interface ".$ip->getIp();
67
					elseif($ip->getNetwork()->getIpv()==4)
Tim Niemeyer's avatar
Tim Niemeyer committed
68
						$command = "ping -c $ping_count -w ".($ping_count+1)*$ping_timeout." -W $ping_timeout ".$ip->getIp();
69
					echo "			".$command."\n";
Tim Niemeyer's avatar
Tim Niemeyer committed
70 71

					PsExecute($command, $ping_hard_timeout, 1);
boombuler's avatar
boombuler committed
72

73 74 75
					//fetch crawl data from router
					$return = array();
					if($ip->getNetwork()->getIpv()==6)
76
						$command = "curl -s --max-time $crawl_timeout -g http://[".$ip->getIp()."%25\$(cat /sys/class/net/$network_connection_ipv6_interface/ifindex)]/node.data | zcat -f";
77
					elseif($ip->getNetwork()->getIpv()==4)
78
						$command = "curl -s --max-time $crawl_timeout -g http://".$ip->getIp()."/node.data | zcat -f";
79 80 81 82 83 84
					echo "			".$command."\n";
					exec($command, $return);
					$return_string = "";
					foreach($return as $string) {
						$return_string .= $string;
					}
boombuler's avatar
boombuler committed
85

86
					//store the crawl data into the database if the router is not offline
Tim Niemeyer's avatar
Tim Niemeyer committed
87 88 89 90 91 92 93 94 95 96
					if(!empty($return_string)) {
						echo "			Craw was successfull, online\n";
						try {
							$xml = new SimpleXMLElement($return_string);
							$data = simplexml2array($xml);
							$data['router_id'] = $router->getRouterId();
							$data['system_data']['status'] = "online";
						} catch (Exception $e) {
							echo nl2br($e->getMessage());
							echo "			There was an error parsing the crawled XML\n";
97 98 99
							$data = array();
							$data['router_id'] = $router->getRouterId();
							$data['system_data']['status'] = "unknown";
100
						}
boombuler's avatar
boombuler committed
101

102 103 104
						/**Insert Router System Data*/
						echo "			Inserting RouterStatus into DB\n";
						$router_status = New RouterStatus(false, (int)$actual_crawl_cycle, $router->getRouterId(),
105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
                            $data['system_data']['status'],
                            false,
                            isset($data['system_data']['hostname']) ? $data['system_data']['hostname'] : "",
                            isset($data['client_count']) ? (int)$data['client_count'] : 0,
                            isset($data['system_data']['chipset']) ? $data['system_data']['chipset'] : "",
                            isset($data['system_data']['cpu']) ? $data['system_data']['cpu'] : "",
                            isset($data['system_data']['memory_total']) ? (int)$data['system_data']['memory_total'] : 0,
                            isset($data['system_data']['memory_caching']) ? (int)$data['system_data']['memory_caching'] : 0,
                            isset($data['system_data']['memory_buffering']) ? (int)$data['system_data']['memory_buffering'] : 0,
                            isset($data['system_data']['memory_free']) ? (int)$data['system_data']['memory_free'] : 0,
                            isset($data['system_data']['loadavg']) ? $data['system_data']['loadavg'] : "",
                            isset($data['system_data']['processes']) ? $data['system_data']['processes'] : "",
                            isset($data['system_data']['uptime']) ? $data['system_data']['uptime'] : "",
                            isset($data['system_data']['idletime']) ? $data['system_data']['idletime'] : "",
                            isset($data['system_data']['local_time']) ? $data['system_data']['local_time'] : "",
                            isset($data['system_data']['distname']) ? $data['system_data']['distname'] : "",
                            isset($data['system_data']['distversion']) ? $data['system_data']['distversion'] : "",
                            isset($data['system_data']['openwrt_core_revision']) ? $data['system_data']['openwrt_core_revision'] : "",
                            isset($data['system_data']['openwrt_feeds_packages_revision']) ? $data['system_data']['openwrt_feeds_packages_revision'] : "",
                            isset($data['system_data']['firmware_version']) ? $data['system_data']['firmware_version'] : "",
                            isset($data['system_data']['firmware_revision']) ? $data['system_data']['firmware_revision'] : "",
                            isset($data['system_data']['kernel_version']) ? $data['system_data']['kernel_version'] : "",
                            isset($data['system_data']['configurator_version']) ? $data['system_data']['configurator_version'] : "",
                            isset($data['system_data']['nodewatcher_version']) ? $data['system_data']['nodewatcher_version'] : "",
                            isset($data['system_data']['fastd_version']) ? $data['system_data']['fastd_version'] : "",
                            isset($data['system_data']['batman_advanced_version']) ? $data['system_data']['batman_advanced_version'] : "");
131
						if($router_status->store()) {
132 133 134 135
							echo "			Inserting Batman advanced interfaces into DB\n";
							/**Insert Batman advanced Interfaces*/
							foreach($data['batman_adv_interfaces'] as $bat_adv_int) {
								try {
boombuler's avatar
boombuler committed
136 137 138 139 140
									$stmt = DB::getInstance()->prepare("INSERT INTO crawl_batman_advanced_interfaces (router_id, crawl_cycle_id, name, status, crawl_date)
															 VALUES (:router_id, :actual_crawl_cycle, :name, :status, NOW());");
									$stmt->execute(array(
										':router_id' => $data['router_id'],
										':actual_crawl_cycle' => $actual_crawl_cycle,
141 142
										':name' => isset($bat_adv_int['name']) ? $bat_adv_int['name'] : "",
										':status' => isset($bat_adv_int['status']) ? $bat_adv_int['status'] : ""
boombuler's avatar
boombuler committed
143
									));
144 145 146 147
								} catch(PDOException $e) {
									echo $e->getMessage();
								}
							}
boombuler's avatar
boombuler committed
148

149 150 151 152
							echo "			Inserting Batman advanced originators into DB\n";
							/**Insert Batman Advanced Originators*/
							$originator_count=count($data['batman_adv_originators']);
							RrdTool::updateRouterBatmanAdvOriginatorsCountHistory($data['router_id'], $originator_count);
boombuler's avatar
boombuler committed
153

154 155 156 157 158 159 160
							$average_link_quality = 0;
							if(!empty($data['batman_adv_originators'])) {
								foreach($data['batman_adv_originators'] as $originator) {
									if(ConfigLine::configByName('crawl_direct_originators_only')=='true' AND
										$originator['originator'] == $originator['nexthop']) {
										$originator_status = new OriginatorStatus(false, (int)$actual_crawl_cycle, (int)$data['router_id'], $originator['originator'],
																				  (int)$originator['link_quality'], $originator['nexthop'], $originator['outgoing_interface'],
Tim Niemeyer's avatar
Tim Niemeyer committed
161
																					$originator['last_seen']);
162 163 164 165 166 167 168 169
										$originator_status->store();
										RrdTool::updateRouterBatmanAdvOriginatorLinkQuality($data['router_id'], $originator['originator'], $originator['link_quality'], time());
										$average_link_quality=$average_link_quality+$originator['link_quality'];
									}
								}
							}
							$average_link_quality=($average_link_quality/$originator_count);
							RrdTool::updateRouterBatmanAdvOriginatorLinkQuality($data['router_id'], "average", $average_link_quality, time());
boombuler's avatar
boombuler committed
170 171


172 173 174 175 176 177
							echo "			Inserting all other Data into DB\n";
							Crawl::insertCrawlData($data);
						} else {
							echo "			RouterStatus could not be inserted into DB\n";
						}
						break 2;
178
					} else {
Tim Niemeyer's avatar
Tim Niemeyer committed
179
						echo "			Crawl was not successfull trying to ping next address\n";
180 181 182 183 184
					}
				}
			}
		}
	}
boombuler's avatar
boombuler committed
185

186
	echo "The process took ".(time()-$starttime)." seconds\n";
floh1111's avatar
floh1111 committed
187

188
	function simplexml2array($xml) {
189
		if (!is_string($xml) AND !is_array($xml) AND (get_class($xml) == 'SimpleXMLElement')) {
190 191 192
			$attributes = $xml->attributes();
			foreach($attributes as $k=>$v) {
				if ($v) $a[$k] = (string) $v;
floh1111's avatar
floh1111 committed
193
			}
194 195 196 197 198 199
			$x = $xml;
			$xml = get_object_vars($xml);
		}
		if (is_array($xml)) {
			if (count($xml) == 0) return (string) $x; // for CDATA
			foreach($xml as $key=>$value) {
200
				$r[$key] = simplexml2array($value);
floh1111's avatar
floh1111 committed
201
			}
202 203
			if (isset($a)) $r['@attributes'] = $a;    // Attributes
			return $r;
floh1111's avatar
floh1111 committed
204
		}
205 206
		return (string) $xml;
	}
207
?>