SlapdObject directory based configuration method, and slapadd impleme…

…ntation (#382)

https://github.com/python-ldap/python-ldap/pull/382

Doc/spelling_wordlist.txt

@@ -129,8 +129,10 @@ serverctrls
 sessionSourceIp
 sessionSourceName
 sessionTrackingIdentifier
+slapadd
 sizelimit
 slapd
+startup
 stderr
 stdout
 str

Lib/slapdtest/_slapdtest.py

@@ -23,34 +23,33 @@
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 
-# a template string for generating simple slapd.conf file
-SLAPD_CONF_TEMPLATE = r"""
-serverID %(serverid)s
-moduleload back_%(database)s
-%(include_directives)s
-loglevel %(loglevel)s
-allow bind_v2
-
-authz-regexp
-  "gidnumber=%(root_gid)s\\+uidnumber=%(root_uid)s,cn=peercred,cn=external,cn=auth"
-  "%(rootdn)s"
-
-database %(database)s
-directory "%(directory)s"
-suffix "%(suffix)s"
-rootdn "%(rootdn)s"
-rootpw "%(rootpw)s"
-
-TLSCACertificateFile "%(cafile)s"
-TLSCertificateFile "%(servercert)s"
-TLSCertificateKeyFile "%(serverkey)s"
-# ignore missing client cert but fail with invalid client cert
-TLSVerifyClient try
-
-authz-regexp
-    "C=DE, O=python-ldap, OU=slapd-test, CN=([A-Za-z]+)"
-    "ldap://ou=people,dc=local???($1)"
-
+# a template string for generating simple slapd.d file
+SLAPD_CONF_TEMPLATE = r"""dn: cn=config
+objectClass: olcGlobal
+cn: config
+olcServerID: %(serverid)s
+olcLogLevel: %(loglevel)s
+olcAllows: bind_v2
+olcAuthzRegexp: {0}"gidnumber=%(root_gid)s\+uidnumber=%(root_uid)s,cn=peercred,cn=external,cn=auth" "%(rootdn)s"
+olcAuthzRegexp: {1}"C=DE, O=python-ldap, OU=slapd-test, CN=([A-Za-z]+)" "ldap://ou=people,dc=local???($1)"
+olcTLSCACertificateFile: %(cafile)s
+olcTLSCertificateFile: %(servercert)s
+olcTLSCertificateKeyFile: %(serverkey)s
+olcTLSVerifyClient: try
+
+dn: cn=module,cn=config
+objectClass: olcModuleList
+cn: module
+olcModuleLoad: back_%(database)s
+
+dn: olcDatabase=%(database)s,cn=config
+objectClass: olcDatabaseConfig
+objectClass: olcMdbConfig
+olcDatabase: %(database)s
+olcSuffix: %(suffix)s
+olcRootDN: %(rootdn)s
+olcRootPW: %(rootpw)s
+olcDbDirectory: %(directory)s
 """
 
 LOCALHOST = '127.0.0.1'
@@ -175,6 +174,9 @@ class SlapdObject(object):
     manager, the slapd server is shut down and the temporary data store is
     removed.
 
+    :param openldap_schema_files: A list of schema names or schema paths to
+        load at startup. By default this only contains `core`.
+
     .. versionchanged:: 3.1
 
         Added context manager functionality
@@ -187,10 +189,10 @@ class SlapdObject(object):
     slapd_loglevel = 'stats stats2'
     local_host = LOCALHOST
     testrunsubdirs = (
-        'schema',
+        'slapd.d',
     )
     openldap_schema_files = (
-        'core.schema',
+        'core.ldif',
     )
 
     TMPDIR = os.environ.get('TMP', os.getcwd())
@@ -217,8 +219,7 @@ def __init__(self):
         self._port = self._avail_tcp_port()
         self.server_id = self._port % 4096
         self.testrundir = os.path.join(self.TMPDIR, 'python-ldap-test-%d' % self._port)
-        self._schema_prefix = os.path.join(self.testrundir, 'schema')
-        self._slapd_conf = os.path.join(self.testrundir, 'slapd.conf')
+        self._slapd_conf = os.path.join(self.testrundir, 'slapd.d')
         self._db_directory = os.path.join(self.testrundir, "openldap-data")
         self.ldap_uri = "ldap://%s:%d/" % (self.local_host, self._port)
         if HAVE_LDAPI:
@@ -262,6 +263,7 @@ def _find_commands(self):
         self.PATH_LDAPDELETE = self._find_command('ldapdelete')
         self.PATH_LDAPMODIFY = self._find_command('ldapmodify')
         self.PATH_LDAPWHOAMI = self._find_command('ldapwhoami')
+        self.PATH_SLAPADD = self._find_command('slapadd')
 
         self.PATH_SLAPD = os.environ.get('SLAPD', None)
         if not self.PATH_SLAPD:
@@ -292,7 +294,6 @@ def setup_rundir(self):
         os.mkdir(self.testrundir)
         os.mkdir(self._db_directory)
         self._create_sub_dirs(self.testrunsubdirs)
-        self._ln_schema_files(self.openldap_schema_files, self.SCHEMADIR)
 
     def _cleanup_rundir(self):
         """
@@ -337,17 +338,8 @@ def gen_config(self):
         for generating specific static configuration files you have to
         override this method
         """
-        include_directives = '\n'.join(
-            'include "{schema_prefix}/{schema_file}"'.format(
-                schema_prefix=self._schema_prefix,
-                schema_file=schema_file,
-            )
-            for schema_file in self.openldap_schema_files
-        )
         config_dict = {
             'serverid': hex(self.server_id),
-            'schema_prefix':self._schema_prefix,
-            'include_directives': include_directives,
             'loglevel': self.slapd_loglevel,
             'database': self.database,
             'directory': self._db_directory,
@@ -371,29 +363,28 @@ def _create_sub_dirs(self, dir_names):
             self._log.debug('Create directory %s', dir_name)
             os.mkdir(dir_name)
 
-    def _ln_schema_files(self, file_names, source_dir):
-        """
-        write symbolic links to original schema files
-        """
-        for fname in file_names:
-            ln_source = os.path.join(source_dir, fname)
-            ln_target = os.path.join(self._schema_prefix, fname)
-            self._log.debug('Create symlink %s -> %s', ln_source, ln_target)
-            os.symlink(ln_source, ln_target)
-
     def _write_config(self):
-        """Writes the slapd.conf file out, and returns the path to it."""
-        self._log.debug('Writing config to %s', self._slapd_conf)
-        with open(self._slapd_conf, 'w') as config_file:
-            config_file.write(self.gen_config())
-        self._log.info('Wrote config to %s', self._slapd_conf)
+        """Loads the slapd.d configuration."""
+        self._log.debug("importing configuration: %s", self._slapd_conf)
+
+        self.slapadd(self.gen_config(), ["-n0"])
+        ldif_paths = [
+            schema
+            if os.path.exists(schema)
+            else os.path.join(self.SCHEMADIR, schema)
+            for schema in self.openldap_schema_files
+        ]
+        for ldif_path in ldif_paths:
+            self.slapadd(None, ["-n0", "-l", ldif_path])
+
+        self._log.debug("import ok: %s", self._slapd_conf)
 
     def _test_config(self):
         self._log.debug('testing config %s', self._slapd_conf)
         popen_list = [
             self.PATH_SLAPD,
             "-Ttest",
-            "-f", self._slapd_conf,
+            "-F", self._slapd_conf,
             "-u",
             "-v",
             "-d", "config"
@@ -417,8 +408,7 @@ def _start_slapd(self):
             urls.append(self.ldapi_uri)
         slapd_args = [
             self.PATH_SLAPD,
-            '-f', self._slapd_conf,
-            '-F', self.testrundir,
+            '-F', self._slapd_conf,
             '-h', ' '.join(urls),
         ]
         if self._log.isEnabledFor(logging.DEBUG):
@@ -523,10 +513,14 @@ def _cli_popen(self, ldapcommand, extra_args=None, ldap_uri=None,
                    stdin_data=None):  # pragma: no cover
         if ldap_uri is None:
             ldap_uri = self.default_ldap_uri
-        args = [
-            ldapcommand,
-            '-H', ldap_uri,
-        ] + self._cli_auth_args() + (extra_args or [])
+
+        if ldapcommand.split("/")[-1].startswith("ldap"):
+            args = [ldapcommand, '-H', ldap_uri] + self._cli_auth_args()
+        else:
+            args = [ldapcommand, '-F', self._slapd_conf]
+
+        args += (extra_args or [])
+
         self._log.debug('Run command: %r', ' '.join(args))
         proc = subprocess.Popen(
             args, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
@@ -577,6 +571,16 @@ def ldapdelete(self, dn, recursive=False, extra_args=None):
         extra_args.append(dn)
         self._cli_popen(self.PATH_LDAPDELETE, extra_args=extra_args)
 
+    def slapadd(self, ldif, extra_args=None):
+        """
+        Runs slapadd on this slapd instance, passing it the ldif content
+        """
+        self._cli_popen(
+            self.PATH_SLAPADD,
+            stdin_data=ldif.encode("utf-8") if ldif else None,
+            extra_args=extra_args,
+        )
+
     def __enter__(self):
         self.start()
         return self

Tests/t_ldap_syncrepl.py

@@ -19,27 +19,44 @@
 from slapdtest import SlapdObject, SlapdTestCase
 
 # a template string for generating simple slapd.conf file
-SLAPD_CONF_PROVIDER_TEMPLATE = r"""
-serverID %(serverid)s
-moduleload back_%(database)s
-moduleload syncprov
-include "%(schema_prefix)s/core.schema"
-loglevel %(loglevel)s
-allow bind_v2
-
-authz-regexp
-  "gidnumber=%(root_gid)s\\+uidnumber=%(root_uid)s,cn=peercred,cn=external,cn=auth"
-  "%(rootdn)s"
-
-database %(database)s
-directory "%(directory)s"
-suffix "%(suffix)s"
-rootdn "%(rootdn)s"
-rootpw "%(rootpw)s"
-overlay syncprov
-syncprov-checkpoint 100 10
-syncprov-sessionlog 100
-index objectclass,entryCSN,entryUUID eq
+SLAPD_CONF_PROVIDER_TEMPLATE = r"""dn: cn=config
+objectClass: olcGlobal
+cn: config
+olcServerID: %(serverid)s
+olcLogLevel: %(loglevel)s
+olcAllows: bind_v2
+olcAuthzRegexp: {0}"gidnumber=%(root_gid)s\+uidnumber=%(root_uid)s,cn=peercred,cn=external,cn=auth" "%(rootdn)s"
+olcAuthzRegexp: {1}"C=DE, O=python-ldap, OU=slapd-test, CN=([A-Za-z]+)" "ldap://ou=people,dc=local???($1)"
+olcTLSCACertificateFile: %(cafile)s
+olcTLSCertificateFile: %(servercert)s
+olcTLSCertificateKeyFile: %(serverkey)s
+olcTLSVerifyClient: try
+
+dn: cn=module,cn=config
+objectClass: olcModuleList
+cn: module
+olcModuleLoad: back_%(database)s
+olcModuleLoad: syncprov
+
+dn: olcDatabase=%(database)s,cn=config
+objectClass: olcDatabaseConfig
+objectClass: olcMdbConfig
+olcDatabase: %(database)s
+olcSuffix: %(suffix)s
+olcRootDN: %(rootdn)s
+olcRootPW: %(rootpw)s
+olcDbDirectory: %(directory)s
+olcDbIndex: objectclass,entryCSN,entryUUID eq
+
+dn: olcOverlay=syncprov,olcDatabase={1}%(database)s,cn=config
+objectClass: olcOverlayConfig
+objectClass: olcSyncProvConfig
+olcOverlay: syncprov
+olcSpCheckpoint: 100 10
+olcSpSessionlog: 100
+"""
+
+OTHER_CONF = r"""
 """
 
 # Define initial data load, both as an LDIF and as a dictionary.

Tests/t_ldapobject.py

@@ -62,6 +62,25 @@
 
 """
 
+SCHEMA_TEMPLATE = """dn: cn=mySchema,cn=schema,cn=config
+objectClass: olcSchemaConfig
+cn: mySchema
+olcAttributeTypes: ( 1.3.6.1.4.1.56207.1.1.1 NAME 'myAttribute'
+    DESC 'fobar attribute'
+    EQUALITY caseExactMatch
+    ORDERING caseExactOrderingMatch
+    SUBSTR caseExactSubstringsMatch
+    SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
+    SINGLE-VALUE
+    USAGE userApplications
+    X-ORIGIN 'foobar' )
+olcObjectClasses: ( 1.3.6.1.4.1.56207.1.2.2 NAME 'myClass'
+    DESC 'foobar objectclass'
+    SUP top
+    STRUCTURAL
+    MUST myAttribute
+    X-ORIGIN 'foobar' )"""
+
 
 class Test00_SimpleLDAPObject(SlapdTestCase):
     """
@@ -94,6 +113,14 @@ def setUp(self):
     def tearDown(self):
         del self._ldap_conn
 
+    def reset_connection(self):
+        try:
+            del self._ldap_conn
+        except AttributeError:
+            pass
+
+        self._ldap_conn = self._open_ldap_conn(bytes_mode=False)
+
     def test_reject_bytes_base(self):
         base = self.server.suffix
         l = self._ldap_conn
@@ -465,6 +492,22 @@ def test_passwd_s(self):
 
         l.delete_s(dn)
 
+    def test_slapadd(self):
+        with self.assertRaises(ldap.INVALID_DN_SYNTAX):
+            self._ldap_conn.add_s("myAttribute=foobar,ou=Container,%s" % self.server.suffix, [
+                ("objectClass", b'myClass'),
+                ("myAttribute", b'foobar'),
+            ])
+
+        self.server.slapadd(SCHEMA_TEMPLATE, ["-n0"])
+        self.server.restart()
+        self.reset_connection()
+
+        self._ldap_conn.add_s("myAttribute=foobar,ou=Container,%s" % self.server.suffix, [
+            ("objectClass", b'myClass'),
+            ("myAttribute", b'foobar'),
+        ])
+
 
 class Test01_ReconnectLDAPObject(Test00_SimpleLDAPObject):
     """
@@ -561,6 +604,11 @@ def tearDown(self):
         del self._sock
         super(Test03_SimpleLDAPObjectWithFileno, self).tearDown()
 
+    def reset_connection(self):
+        self._sock.close()
+        del self._sock
+        super(Test03_SimpleLDAPObjectWithFileno, self).reset_connection()
+
 
 if __name__ == '__main__':
     unittest.main()