Firewall / Shaper: the current number of allowed source / destination addresses is limited to a couple of addresses. in order to match larger lists one need to use tables in order for ipfw to process them.

This commit replaces all rule source / destinations where more than one address is being targetted with a tablename like __rule__{uuid}__[source,destination] a support tool after load flushes the selected addresses into the table (which is automatically created by the ruleset). closes https://github.com/opnsense/core/issues/4756

src/etc/rc.ipfw

@@ -33,6 +33,8 @@ sysctl net.inet.ip.dummynet.hash_size=256 > /dev/null
 
 # reload ipfw rules
 ipfw -q /usr/local/etc/ipfw.rules
+# load tables
+/usr/local/opnsense/scripts/shaper/update_tables
 
 if [ ! -f /tmp/ipfw.firstload ]; then
 	# we need to make sure ipfw is loaded as last

src/opnsense/scripts/shaper/update_tables

@@ -0,0 +1,56 @@
+#!/usr/local/bin/python3
+
+"""
+    Copyright (c) 2021 Ad Schellevis <ad@opnsense.org>
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+"""
+import os
+import subprocess
+from configparser import ConfigParser
+
+
+if __name__ == '__main__':
+    all_tables = dict()
+    updater_conf = '/usr/local/etc/ipfw_rule_tables.conf'
+    if os.path.exists(updater_conf):
+        cnf = ConfigParser()
+        cnf.read(updater_conf)
+        for section in cnf.sections():
+            if cnf.has_option(section, 'table_name') and cnf.has_option(section, 'table_content'):
+                all_tables[cnf.get(section, 'table_name')] = cnf.get(section, 'table_content').split(',')
+
+    ipfw_list_cmd = ['/sbin/ipfw', 'table', 'all', 'list']
+    for line in subprocess.run(ipfw_list_cmd, capture_output=True, text=True).stdout.split('\n'):
+        if line.startswith('---'):
+            table_name = line.split('(')[1].split(')')[0]
+            if table_name.startswith('__rule__') and table_name not in all_tables:
+                subprocess.run(['/sbin/ipfw', 'table', table_name, 'destroy'], capture_output=True)
+
+    for table_name in all_tables:
+        # XXX: a full diff of the current table contents might be better, but for shaping purposes we likely won't
+        #      notice the glitch
+        subprocess.run(['/sbin/ipfw', 'table', table_name, 'flush'], capture_output=True)
+        for address in all_tables[table_name]:
+            subprocess.run(['/sbin/ipfw', 'table', table_name, 'add', address], capture_output=True)

src/opnsense/service/templates/OPNsense/IPFW/+TARGETS

@@ -1,2 +1,3 @@
 rc.conf.d:/etc/rc.conf.d/ipfw
 ipfw.conf:/usr/local/etc/ipfw.rules
+ipfw_rule_tables.conf:/usr/local/etc/ipfw_rule_tables.conf

src/opnsense/service/templates/OPNsense/IPFW/ipfw.conf

@@ -1,5 +1,6 @@
 {# Macro import #}
 {% from 'OPNsense/Macros/interface.macro' import physical_interface %}
+{% from 'OPNsense/IPFW/rules.macro' import convert_address %}
 {# collect interfaces list (with / without captive portal enabled) #}
 {% set cp_interface_list = [] %}
 {% set no_cp_interface_list = [] %}
@@ -166,8 +167,8 @@ add 60000 return via any
 {%                         if rule.direction == 'in' or not rule.direction %}
 add {{loop.index + 60000}} {{ helpers.getUUIDtag(rule.target) }} {{
     helpers.getUUID(rule.target).number }} {{ rule.proto.split('_')[0] }} from {%
-    if rule.source_not|default('0') == '1' %}not {% endif %}{{ rule.source }} to {%
-    if rule.destination_not|default('0') == '1' %}not {% endif %}{{rule.destination
+    if rule.source_not|default('0') == '1' %}not {% endif %}{{ convert_address(rule, 'source') }} to {%
+    if rule.destination_not|default('0') == '1' %}not {% endif %}{{convert_address(rule, 'destination')
     }} src-port  {{ rule.src_port }} dst-port {{ rule.dst_port }} recv {{
     physical_interface(rule.interface) }} {%
     if rule.proto.split('_')[1]|default('') == 'ack' %} {{ rule.proto.split('_')[2]|default('') }} tcpflags ack {% endif %}{%
@@ -179,8 +180,8 @@ add {{loop.index + 60000}} {{ helpers.getUUIDtag(rule.target) }} {{
 {%                         if rule.direction == 'out' or not rule.direction %}
 add {{loop.index + 60000}} {{ helpers.getUUIDtag(rule.target) }} {{
     helpers.getUUID(rule.target).number }} {{ rule.proto.split('_')[0] }} from {%
-    if rule.source_not|default('0') == '1' %}not {% endif %}{{ rule.source }} to {%
-    if rule.destination_not|default('0') == '1' %}not {% endif %}{{rule.destination
+    if rule.source_not|default('0') == '1' %}not {% endif %}{{ convert_address(rule, 'source') }} to {%
+    if rule.destination_not|default('0') == '1' %}not {% endif %}{{convert_address(rule, 'destination')
     }} src-port  {{ rule.src_port }} dst-port {{ rule.dst_port }} xmit {{
     physical_interface(rule.interface) }} {%
     if rule.proto.split('_')[1]|default('') == 'ack' %} {{ rule.proto.split('_')[2]|default('') }} tcpflags ack {% endif %}{%
@@ -193,8 +194,8 @@ add {{loop.index + 60000}} {{ helpers.getUUIDtag(rule.target) }} {{
 {#  normal, single interface situation  #}
 add {{loop.index + 60000}} {{ helpers.getUUIDtag(rule.target) }} {{
     helpers.getUUID(rule.target).number }} {{ rule.proto.split('_')[0] }} from {%
-    if rule.source_not|default('0') == '1' %}not {% endif %}{{ rule.source }} to {%
-    if rule.destination_not|default('0') == '1' %}not {% endif %}{{rule.destination
+    if rule.source_not|default('0') == '1' %}not {% endif %}{{ convert_address(rule, 'source') }} to {%
+    if rule.destination_not|default('0') == '1' %}not {% endif %}{{convert_address(rule, 'destination')
     }} src-port  {{ rule.src_port }} dst-port {{ rule.dst_port }} {{rule.direction}} {%
     if rule.proto.split('_')[1]|default('') == 'ack' %}{{ rule.proto.split('_')[2]|default('') }} tcpflags ack {% endif %} {%
     if rule.iplen|default('') != '' %} iplen 1-{{ rule.iplen }}{% endif %}{%

src/opnsense/service/templates/OPNsense/IPFW/ipfw_rule_tables.conf

@@ -0,0 +1,14 @@
+{% if helpers.exists('OPNsense.TrafficShaper.rules.rule') %}
+{%     for rule in helpers.toList('OPNsense.TrafficShaper.rules.rule', 'sequence', 'int') %}
+{%         if rule.source.find(',') > -1 %}
+[rule_{{loop.index}}_source]
+table_name=__rule__{{ rule['@uuid'] }}__source
+table_content={{rule.source}}
+{%         endif %}
+{%         if rule.destination.find(',') > -1 %}
+[rule_{{loop.index}}_destination]
+table_name=__rule__{{ rule['@uuid'] }}__destination
+table_content={{rule.destination}}
+{%         endif %}
+{%     endfor %}
+{% endif %}

src/opnsense/service/templates/OPNsense/IPFW/rules.macro

@@ -0,0 +1,6 @@
+{% macro convert_address(rule, attribute) -%}
+{% if rule[attribute] and rule[attribute].find(',') > -1
+%}table(__rule__{{ rule['@uuid'] }}__{{attribute}}){%
+else %}{{rule[attribute]}}{%
+endif
+%}{%- endmacro %}