Commit 3dd0a97
committed
1 parent 94cde64 commit 3dd0a97
File tree
21 files changed
+417
-184
lines changed- Modules
- _hashlib
- PCbuild
21 files changed
+417-184lines changedLines changed: 28 additions & 13 deletionsOriginal file line number Diff line number Diff line change @@ -1481,6 +1481,12 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS)14811481-rm -f $@14821482$(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS)148314831484+##########################################################################1485+# '_hashlib', '_hmac' and HACL*-based modules helpers1486+1487+Modules/_hashlib/hashlib_buffer.o: $(srcdir)/Modules/_hashlib/hashlib_buffer.c $(srcdir)/Modules/_hashlib/hashlib_buffer.h $(PYTHON_HEADERS)1488+$(CC) -I$(srcdir)/Modules/_hashlib -c $(PY_STDMODULE_CFLAGS) $(CCSHARED) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c1489+14841490##########################################################################14851491# HACL* library build14861492#@@ -3323,22 +3329,31 @@ MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h33233329MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@33243330MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@33253331MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@3326-MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h33273332MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h332833333334+MODULE__HASHLIB_DEPS= \3335+$(srcdir)/Modules/_hashlib/hashlib_buffer.h \3336+$(srcdir)/Modules/_hashlib/hashlib_fetch.h \3337+$(srcdir)/Modules/_hashlib/hashlib_mutex.h3338+3339+MODULE__HASHLIB_LDEPS= \3340+Modules/_hashlib/hashlib_buffer.o \3341+Modules/_hashlib/hashlib_fetch.o3342+33293343# HACL*-based cryptographic primitives3330-MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@)3331-MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@)3332-MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@)3333-MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@)3334-MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@)3335-MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@)3336-MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@)3337-MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@)3338-MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@)3339-MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@)3340-MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@)3341-MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@)3344+MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@)3345+MODULE__MD5_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@)3346+MODULE__SHA1_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@)3347+MODULE__SHA1_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@)3348+MODULE__SHA2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@)3349+MODULE__SHA2_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@)3350+MODULE__SHA3_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@)3351+MODULE__SHA3_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@)3352+MODULE__BLAKE2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@)3353+MODULE__BLAKE2_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@)3354+3355+MODULE__HMAC_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@)3356+MODULE__HMAC_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@)3342335733433358MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c33443359MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.hLines changed: 40 additions & 0 deletionsOriginal file line number Diff line number Diff line change @@ -0,0 +1,40 @@1+#include "hashlib_buffer.h"2+3+int4+_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string)5+{6+if (data != NULL && string == NULL) {7+/ called as H(data) or H(data=...)8+*res = data;9+return 1;10+}11+else if (data == NULL && string != NULL) {12+/ called as H(string=...)13+if (PyErr_WarnEx(PyExc_DeprecationWarning,14+"the 'string' keyword parameter is deprecated since "15+"Python 3.15 and slated for removal in Python 3.19; "16+"use the 'data' keyword parameter or pass the data "17+"to hash as a positional argument instead", 1) < 0)18+{19+*res = NULL;20+return -1;21+}22+*res = string;23+return 1;24+}25+else if (data == NULL && string == NULL) {26+/ fast path when no data is given27+assert(!PyErr_Occurred());28+*res = NULL;29+return 0;30+}31+else {32+/ called as H(data=..., string)33+*res = NULL;34+PyErr_SetString(PyExc_TypeError,35+"'data' and 'string' are mutually exclusive "36+"and support for 'string' keyword parameter "37+"is slated for removal in a future version.");38+return -1;39+}40+}Lines changed: 60 additions & 0 deletionsOriginal file line number Diff line number Diff line change @@ -0,0 +1,60 @@1+#ifndef _HASHLIB_HASHLIB_BUFFER_H2+#define _HASHLIB_HASHLIB_BUFFER_H3+4+#include "Python.h"5+6+/*7+* Given an buffer-like OBJ, fill in the buffer VIEW with the result8+* of PyObject_GetBuffer.9+*10+* On error, set an exception and execute the ERRACTION statements,11+* e.g. 'return NULL' or 'goto error'.12+*13+* Parameters14+*15+* OBJ An object supporting the buffer API.16+* VIEW A Py_buffer pointer to fill.17+* ERRACTION The statements to execute on error.18+*/19+#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION) \20+do { \21+if (PyUnicode_Check((OBJ))) { \22+PyErr_SetString(PyExc_TypeError, \23+"strings must be encoded before hashing"); \24+ERRACTION; \25+} \26+if (!PyObject_CheckBuffer((OBJ))) { \27+PyErr_SetString(PyExc_TypeError, \28+"object supporting the buffer API required"); \29+ERRACTION; \30+} \31+if (PyObject_GetBuffer((OBJ), (VIEW), PyBUF_SIMPLE) == -1) { \32+ERRACTION; \33+} \34+if ((VIEW)->ndim > 1) { \35+PyErr_SetString(PyExc_BufferError, \36+"buffer must be one-dimensional"); \37+PyBuffer_Release((VIEW)); \38+ERRACTION; \39+} \40+} while(0)41+42+/* Specialization of GET_BUFFER_VIEW_OR_ERROR() returning NULL on error. */43+#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW) \44+GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL)45+46+/*47+* Allow to use the 'data' or 'string' keyword in hashlib.new()48+* and other hash functions named constructors.49+*50+* - If 'data' and 'string' are both non-NULL, set an exception and return -1.51+* - If 'data' and 'string' are both NULL, set '*res' to NULL and return 0.52+* - Otherwise, set '*res' to 'data' or 'string' and return 1. A deprecation53+* warning is set when 'string' is specified.54+*55+* The symbol is exported for '_hashlib' and HACL*-based extension modules.56+*/57+PyAPI_FUNC(int)58+_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string);59+60+#endif / !_HASHLIB_HASHLIB_BUFFER_HLines changed: 1 addition & 0 deletionsOriginal file line number Diff line number Diff line change @@ -0,0 +1 @@1+#include "hashlib_fetch.h"Lines changed: 130 additions & 0 deletionsOriginal file line number Diff line number Diff line change @@ -0,0 +1,130 @@1+/*2+* Interface for fetching a message digest from a digest-like identifier.3+*4+* The following table summaries the possible algorthms:5+*6+* +----------+--------------+--------------+---------------------------------+7+* | Family | Algorithm | Python Name | Notes |8+* +==========+==============+==============+=================================+9+* | MD @ |10+* | +--------------+--------------+---------------------------------+11+* | | MD5 | "md5" | |12+* +----------+--------------+--------------+---------------------------------+13+* | SHA1 @ |14+* | +--------------+--------------+---------------------------------+15+* | | SHA1-160 | "sha1" | |16+* +----------+--------------+--------------+---------------------------------+17+* | SHA2 @ |18+* | +--------------+--------------+---------------------------------+19+* | | SHA2-224 | "sha224" | |20+* | | SHA2-256 | "sha256" | |21+* | | SHA2-384 | "sha384" | |22+* | | SHA2-512 | "sha512" | |23+* +----------+--------------+--------------+---------------------------------+24+* | SHA2t @ Truncated SHA2-512 |25+* | +--------------+--------------+---------------------------------+26+* | | SHA2-512/224 | "sha512_224" | |27+* | | SHA2-512/256 | "sha512_256" | |28+* +----------+--------------+--------------+---------------------------------+29+* | SHA3 @ |30+* | +--------------+--------------+---------------------------------+31+* | | SHA3-224 | "sha3_224" | |32+* | | SHA3-256 | "sha3_256" | |33+* | | SHA3-384 | "sha3_384" | |34+* | | SHA3-512 | "sha3_512" | |35+* +----------+--------------+--------------+---------------------------------+36+* | SHA3-XOF @ Extensible Output Functions |37+* | +--------------+--------------+---------------------------------+38+* | | SHAKE-128 | "shake_128" | |39+* | | SHAKE-256 | "shake_256" | |40+* +----------+--------------+--------------+---------------------------------+41+* | BLAKE2 @ |42+* | +--------------+--------------+---------------------------------+43+* | | BLAKE2b | "blake2b" | |44+* | | BLAKE2s | "blake2s" | |45+* +----------+--------------+--------------+---------------------------------+46+*/47+48+#ifndef _HASHLIB_HASHLIB_FETCH_H49+#define _HASHLIB_HASHLIB_FETCH_H50+51+#include "Python.h"52+53+#define Py_HASHLIB_MD_NS(ATTR) Py_hashlib_message_digest_ ## ATTR54+#define Py_HASHLIB_MD_FAMILY(FAMILY_ID) Py_HASHLIB_MD_NS(family_ ## FAMILY_ID)55+#define Py_HASHLIB_MD_MEMBER(MEMBER_ID) Py_HASHLIB_MD_NS(member_ ## MEMBER_ID)56+57+#define Py_HASHLIB_MD_NAMES Py_HASHLIB_MD_NS(NAMES)58+#define Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(Py_HASHLIB_MD_NAMES)59+#define Py_HASHLIB_MD_NAME(MEMBER_ID) \60+( \61+assert(Py_HASHLIB_MD_NAME(MEMBER_ID) < Py_HASHLIB_MD_COUNT), \62+Py_HASHLIB_MD_NAMES[Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \63+)64+65+typedef enum {66+Py_HASHLIB_MD_FAMILY(MD) = 0,67+Py_HASHLIB_MD_FAMILY(SHA1),68+Py_HASHLIB_MD_FAMILY(SHA2),69+Py_HASHLIB_MD_FAMILY(SHA2t),70+Py_HASHLIB_MD_FAMILY(SHA3),71+Py_HASHLIB_MD_FAMILY(SHA3_XOF),72+Py_HASHLIB_MD_FAMILY(BLAKE2),73+} Py_HASHLIB_MD_NS(family);74+75+typedef enum {76+/* MD-family */77+Py_HASHLIB_MD_MEMBER(md5) = 0,78+/* SHA-1 family */79+Py_HASHLIB_MD_MEMBER(sha1),80+/* SHA-2 family */81+Py_HASHLIB_MD_MEMBER(sha224),82+Py_HASHLIB_MD_MEMBER(sha256),83+Py_HASHLIB_MD_MEMBER(sha384),84+Py_HASHLIB_MD_MEMBER(sha512),85+/* Truncated SHA-2 family */86+Py_HASHLIB_MD_MEMBER(sha512_224),87+Py_HASHLIB_MD_MEMBER(sha512_256),88+/* SHA-3 family */89+Py_HASHLIB_MD_MEMBER(sha3_224),90+Py_HASHLIB_MD_MEMBER(sha3_256),91+Py_HASHLIB_MD_MEMBER(sha3_384),92+Py_HASHLIB_MD_MEMBER(sha3_512),93+/* SHA-3 XOF SHAKE family */94+Py_HASHLIB_MD_MEMBER(shake_128),95+Py_HASHLIB_MD_MEMBER(shake_256),96+/* BLAKE-2 family */97+Py_HASHLIB_MD_MEMBER(blake2b),98+Py_HASHLIB_MD_MEMBER(blake2s),99+} Py_HASHLIB_MD_NS(member);100+101+static const char *Py_HASHLIB_MD_NAMES[] = {102+#define DECL_MESSAGE_DIGEST_NAME(ID) [Py_HASHLIB_MD_MEMBER(ID)] = #ID103+/* MD-family */104+DECL_MESSAGE_DIGEST_NAME(md5),105+/* SHA-1 family */106+DECL_MESSAGE_DIGEST_NAME(sha1),107+/* SHA-2 family */108+DECL_MESSAGE_DIGEST_NAME(sha224),109+DECL_MESSAGE_DIGEST_NAME(sha256),110+DECL_MESSAGE_DIGEST_NAME(sha384),111+DECL_MESSAGE_DIGEST_NAME(sha512),112+/* Truncated SHA-2 family */113+DECL_MESSAGE_DIGEST_NAME(sha512_224),114+DECL_MESSAGE_DIGEST_NAME(sha512_256),115+/* SHA-3 family */116+DECL_MESSAGE_DIGEST_NAME(sha3_224),117+DECL_MESSAGE_DIGEST_NAME(sha3_256),118+DECL_MESSAGE_DIGEST_NAME(sha3_384),119+DECL_MESSAGE_DIGEST_NAME(sha3_512),120+/* SHA-3 XOF SHAKE family */121+DECL_MESSAGE_DIGEST_NAME(shake_128),122+DECL_MESSAGE_DIGEST_NAME(shake_256),123+/* BLAKE-2 family */124+DECL_MESSAGE_DIGEST_NAME(blake2b),125+DECL_MESSAGE_DIGEST_NAME(blake2s),126+#undef DECL_MESSAGE_DIGEST_NAME127+NULL /* sentinel */128+};129+130+#endif / !_HASHLIB_HASHLIB_FETCH_HLines changed: 62 additions & 0 deletionsOriginal file line number Diff line number Diff line change @@ -0,0 +1,62 @@1+#ifndef _HASHLIB_HASHLIB_MUTEX_H2+#define _HASHLIB_HASHLIB_MUTEX_H3+4+#include "Python.h"5+#include "pycore_lock.h" / PyMutex6+7+/*8+* Maximum number of bytes for a message for which the GIL is held9+* when performing incremental hashing.10+*/11+#define HASHLIB_GIL_MINSIZE 204812+13+/*14+* Helper code to synchronize access to the hash object when the GIL is15+* released around a CPU consuming hashlib operation. All code paths that16+* access a mutable part of obj must be enclosed in an ENTER_HASHLIB /17+* LEAVE_HASHLIB block or explicitly acquire and release the lock inside18+* a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for19+* an operation.20+*21+* These only drop the GIL if the lock acquisition itself is likely to22+* block. Thus the non-blocking acquire gating the GIL release for a23+* blocking lock acquisition. The intent of these macros is to surround24+* the assumed always "fast" operations that you aren't releasing the25+* GIL around. Otherwise use code similar to what you see in hash26+* function update() methods.27+*/28+29+/* Prevent undefined behaviors via multiple threads entering the C API. */30+#define HASHLIB_LOCK_HEAD \31+bool use_mutex; \32+PyMutex mutex;33+34+#ifdef Py_GIL_DISABLED35+#define HASHLIB_INIT_MUTEX(OBJ) \36+do { \37+(OBJ)->mutex = (PyMutex){0}; \38+(OBJ)->use_mutex = true; \39+} while (0)40+#else41+#define HASHLIB_INIT_MUTEX(OBJ) \42+do { \43+(OBJ)->mutex = (PyMutex){0}; \44+(OBJ)->use_mutex = false; \45+} while (0)46+#endif47+48+#define ENTER_HASHLIB(OBJ) \49+do { \50+if ((OBJ)->use_mutex) { \51+PyMutex_Lock(&(OBJ)->mutex); \52+} \53+} while (0)54+55+#define LEAVE_HASHLIB(OBJ) \56+do { \57+if ((OBJ)->use_mutex) { \58+PyMutex_Unlock(&(OBJ)->mutex); \59+} \60+} while (0)61+62+#endif / !_HASHLIB_HASHLIB_MUTEX_HLines changed: 5 additions & 8 deletionsOriginal file line number Diff line number Diff line change @@ -26,7 +26,8 @@2626#include "pycore_hashtable.h"2727#include "pycore_strhex.h" / _Py_strhex()2828#include "pycore_pyatomic_ft_wrappers.h" / FT_ATOMIC_LOAD_PTR_RELAXED29-#include "hashlib.h"29+#include "_hashlib/hashlib_buffer.h"30+#include "_hashlib/hashlib_mutex.h"30313132/* EVP is the preferred interface to hashing in OpenSSL */3233#include <openssl/evp.h>@@ -279,20 +280,16 @@ get_hashlib_state(PyObject *module)279280280281typedef struct {281282PyObject_HEAD283+HASHLIB_LOCK_HEAD282284EVP_MD_CTX *ctx; /* OpenSSL message digest context */283-/ Prevents undefined behavior via multiple threads entering the C API.284-bool use_mutex;285-PyMutex mutex; /* OpenSSL context lock */286285} HASHobject;287286288287#define HASHobject_CAST(op) ((HASHobject *)(op))289288290289typedef struct {291290PyObject_HEAD291+HASHLIB_LOCK_HEAD292292HMAC_CTX *ctx; /* OpenSSL hmac context */293-/ Prevents undefined behavior via multiple threads entering the C API.294-bool use_mutex;295-PyMutex mutex; /* HMAC context lock */296293} HMACobject;297294298295#define HMACobject_CAST(op) ((HMACobject *)(op))@@ -1126,7 +1123,7 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj,1126112311271124if (view.buf && view.len) {11281125if (view.len >= HASHLIB_GIL_MINSIZE) {1129-/* We do not initialize self->lock here as this is the constructor1126+/* Do not initialize self->mutex here as this is the constructor11301127* where it is not yet possible to have concurrent access. */11311128Py_BEGIN_ALLOW_THREADS11321129result = _hashlib_HASH_hash(self, view.buf, view.len);
0 commit comments