IP to Geo

IP to Geo Git Source Tree

Root/data/build_c_array.py

1#!/usr/bin/env python
2#-*- coding: utf-8
3
4import sys
5
6COUNTRY_CODE_INDEX=1
7IP_TYPE_INDEX=2
8IP_INDEX=3
9IP_SIZE_INDEX=4
10
11class IP_ELEMENT(object):
12 def __init__(self, start, end=None, size=0, country_code=None, level=0):
13 self._start = start
14 self._end = end
15 self._size = size
16 self._country_code = country_code
17 self._prev = None
18 self._next = None
19 self._childs = None
20 self._average = 0
21 self._level = level
22
23 if not self._end: self._compute_last_ip()
24
25 self._splitted_start = self.split_ip(self._start)
26 self._splitted_end = self.split_ip(self._end)
27
28 def split_ip(self, ip):
29 return [int(x, self._base) for x in ip.split(self._separator)]
30
31 def ip_to_str(self, int_ip):
32 res = []
33 for i in range(0, self.get_ip_len()):
34 res.insert(0, self._format % int((int_ip >> (i*8)) & 0xFF))
35 return self._separator.join(res)
36
37 def ip_array_to_int(self, array):
38 val = 0
39 for i in range(0, len(array)):
40 val += array[len(array)-i-1] << (i*8)
41 return val
42
43 def ip_to_int(self, str_ip):
44 return self.ip_array_to_int(self.split_ip(str_ip))
45
46 def make_group(self):
47 ip_val = self._splitted_start[::]
48 for i in range(self._level+1, self.get_ip_len()):
49 ip_val[i] = 0
50 return self._separator.join([self._format % x for x in ip_val])
51
52 def name(self):
53 return 'ip__%s__%s' %(self._start.replace(self._separator, '_'), self._end.replace(self._separator, '_'))
54
55 def _compute_last_ip(self):
56 raise NotImplementedError()
57
58 def set_next(self, ip):
59 self._next = ip
60
61 def set_prev(self, ip):
62 self._prev = ip
63
64 def set_childs(self, ip):
65 self._childs = ip
66
67 def set_average(self, average):
68 self._average = average
69
70 def set_level(self, level):
71 self._level = level
72
73 def printme(self):
74 print 'static const ip_level %s = {' % (self.name())
75 print '\t.prev = %s,' % (self._prev and '&%s' % (self._prev.name()) or 'NULL')
76 print '\t.next = %s,' % (self._next and '&%s' % (self._next.name()) or 'NULL')
77 print '\t.childs = %s,' % (self._childs and '&%s' % (self._childs.name()) or 'NULL')
78 print '\t.start = %d,' % (self._splitted_start[self._level])
79 print '\t.end = %d,' % (self._splitted_end[self._level])
80 print '\t.average = %d,' % (self._average)
81 print '\t.code = %d,' % (self._country_code and self._country_code or 0)
82 print '};'
83
84 def get_ip_len(self):
85 raise NotImplementedError()
86
87class IP_ELEMENT4(IP_ELEMENT):
88
89 def __init__(self, start, end=None, size=0, country_code=None, level=0):
90 self._separator = '.'
91 self._base = 10
92 self._format = '%d'
93 super(IP_ELEMENT4, self).__init__(start, end, size, country_code, level)
94
95 def get_ip_len(self):
96 return 4
97
98 def _compute_last_ip(self):
99 size = self._size
100 end_ip = self.ip_to_int(self._start)
101 i=0
102 while size > 0:
103 end_ip += (((size % 256)-1) & 0xFF) << (i*8)
104 size = int(size/256)
105 i += 1
106 self._end = self.ip_to_str(end_ip)
107 # print '%s + %d -> %s' % (self._start, self._size, self._end)
108
109class IP_ELEMENT6(IP_ELEMENT):
110
111 def __init__(self, start, end=None, size=0, country_code=None, level=0):
112 self._separator = ':'
113 self._base = 16
114 self._format = '%02x'
115 super(IP_ELEMENT6, self).__init__(start, end, size, country_code, level)
116
117 def get_ip_len(self):
118 return 16
119
120 def _get_mask(self):
121 mask = 0
122 for i in range(0, self._size):
123 mask += 1 << i
124 mask <<= 128-self._size
125 return mask
126
127 def _compute_last_ip(self):
128 if self._size == 0:
129 self._end = self._start[:]
130 else:
131 mask = self._get_mask()
132 self._end = self.ip_to_str(self.ip_to_int(self._start) | ~mask)
133
134def extend_ipv6(ipv6):
135 tmp = ''
136 for s in ipv6.split(':'):
137 if not s: break
138 while len(s) != 4:
139 s = '0' + s
140 tmp += s
141 while len(tmp) < 16*2:
142 tmp += '0'
143 res = ''
144 for i in range(0, 15*2, 2):
145 res += tmp[i] + tmp[i+1] + ':'
146 res += tmp[30] + tmp[31]
147 return res
148
149countries = []
150
151f = open("prefix_res")
152array_vals_ipv4 = {}
153array_vals_ipv6 = {}
154while True:
155 l = f.readline()
156 # l = sys.stdin.readline()
157 if not l: break
158
159 information = l.split('|')
160 country = information[COUNTRY_CODE_INDEX].lower()
161 if not country: continue # Available or reserved but not assigned
162
163 try:
164 country_idx = countries.index(country)
165 except ValueError:
166 country_idx = len(countries)
167 countries.append(country)
168
169 ip = information[IP_INDEX]
170 if information[IP_TYPE_INDEX] == 'ipv4':
171 array_vals_ipv4[ip] = IP_ELEMENT4(ip, None, int(information[IP_SIZE_INDEX]), country_idx)
172 elif information[IP_TYPE_INDEX] == 'ipv6':
173 ip = extend_ipv6(ip)
174 array_vals_ipv6[ip] = IP_ELEMENT6(ip, None, int(information[IP_SIZE_INDEX]), country_idx)
175 else:
176 sys.stderr.write('Unknown IP type %s\n' % (information[IP_TYPE_INDEX]))
177
178print '/* This file was automatically generated, do not edit it ! */'
179print '#include <stdint.h>\n\n'
180
181def ip_sort(a, b):
182 for i in range(0, len(a._splitted_start)):
183 if a._splitted_start[i] != b._splitted_start[i]:
184 return a._splitted_start[i] - b._splitted_start[i]
185 return 0
186
187def get_interval(root, intervals, level):
188 new_intervals = []
189 for ip in intervals:
190 if ip._splitted_start[level] != root: break
191 new_intervals.append(ip)
192 return new_intervals
193
194# 1.5.0.0
195# -> 1.5.0.0 .. 1.5.29.0
196# -> 1.5.30.0 .. 1.5.30.128
197# -> 1.5.30.129 .. 1.5.31.0
198# -> 1.5.32.0 .. 1.5.33.0
199# -> 1.6.32.0 .. 1.7.0.0
200
201def print_interval(interval):
202 p = '['
203 for i in interval:
204 p += '%s, ' % (i.name())
205 p += ']'
206 return p
207
208def compute_average(root):
209 total = 0
210 count = 0
211 child = root._childs
212 while child:
213 total += 1
214 count += (child._splitted_end[child._level] - child._splitted_start[child._level] + 1)
215 child = child._next
216 average = int(count/total)
217 # Find highest power of 2 < average
218 for i in range(0, 9):
219 if average < (1 << i):
220 root.set_average(i-1)
221 break
222
223def manage_root(root, intervals, level):
224 cur_start = 0
225 prev = None
226 first = None
227 cur_len = 0
228 if level >= 3: return (0, None)
229 # print 'manage_root(%d, %s, %d)' %\
230 # (root, print_interval(intervals), level)
231 while True:
232 if cur_start >= len(intervals): break
233 cur_ip = intervals[cur_start]
234 sub_interval = get_interval(cur_ip._splitted_start[level],\
235 intervals[cur_start+1:],\
236 level)
237 if sub_interval:
238 cur_ip.set_level(level+1)
239 for ip in sub_interval:
240 ip.set_level(level+1)
241 new_group = cur_ip.__class__(cur_ip.make_group(), level=level)
242 sub_interval.insert(0, cur_ip)
243 child = manage_root(cur_ip._splitted_start[level+1], sub_interval, level+1)
244 new_group.set_childs(cur_ip)
245 compute_average(new_group)
246 cur_ip = new_group
247 cur_start += len(sub_interval)
248 else:
249 cur_ip.set_level(level)
250 cur_start += 1
251
252 cur_ip.set_prev(prev)
253 if (prev): prev.set_next(cur_ip)
254 prev = cur_ip
255 if not first: first = cur_ip
256 return first
257
258def print_ip(ip):
259 cur_ip = ip
260 while cur_ip:
261 if cur_ip._childs:
262 print_ip(cur_ip._childs)
263 print 'static const ip_level %s;' % (cur_ip.name())
264 cur_ip = cur_ip._next
265 print ''
266 cur_ip = ip
267 while cur_ip:
268 cur_ip.printme()
269 cur_ip = cur_ip._next
270
271def build_array(ip_list, array_name):
272 ip_list.sort(ip_sort)
273 start_idx = 0
274 end_idx = start_idx+1
275 cur_interval = [ip_list[start_idx]]
276 root = ip_list[start_idx]._splitted_start[0]
277 root_ips = [None] * 256
278
279 while True:
280 if end_idx >= len(ip_list): break
281 if ip_list[end_idx]._splitted_start[0] != root:
282 start_idx = end_idx
283 res = manage_root(root, cur_interval, 1)
284 print_ip(res)
285 root_ips[res._splitted_start[0]] = res
286 cur_interval = [ip_list[end_idx]]
287 root = ip_list[start_idx]._splitted_start[0]
288 else:
289 cur_interval.append(ip_list[end_idx])
290 end_idx += 1
291 res = manage_root(root, cur_interval, 1)
292 print_ip(res)
293
294 print '\nstatic const ip_level* %s[256] = {' % (array_name)
295 for i in range(0, 256):
296 if root_ips[i]:
297 print '\t&%s,' % (root_ips[i].name())
298 else:
299 print '\tNULL, // %d' % (i)
300 print '};\n'
301
302build_array(array_vals_ipv4.values(), 's_root_ipv4')
303build_array(array_vals_ipv6.values(), 's_root_ipv6')
304
305print 'static const uint8_t country_codes[][3] = {'
306for cc in countries:
307 print '\t{"%s"},' % (cc)
308print '};\n'

Archive Download this file

Branches

Tags