#include <config.h>
#include <stddef.h>
#include <isc/atomic.h>
#include <isc/magic.h>
#include <isc/msgs.h>
#include <isc/platform.h>
#include <isc/rwlock.h>
#include <isc/util.h>
#define RWLOCK_MAGIC ISC_MAGIC('R', 'W', 'L', 'k')
#define VALID_RWLOCK(rwl) ISC_MAGIC_VALID(rwl, RWLOCK_MAGIC)
#ifdef ISC_PLATFORM_USETHREADS
#ifndef RWLOCK_DEFAULT_READ_QUOTA
#define RWLOCK_DEFAULT_READ_QUOTA 4
#endif
#ifndef RWLOCK_DEFAULT_WRITE_QUOTA
#define RWLOCK_DEFAULT_WRITE_QUOTA 4
#endif
#ifdef ISC_RWLOCK_TRACE
#include <stdio.h>
#include <isc/thread.h>
static void
print_lock(const char *operation, isc_rwlock_t *rwl, isc_rwlocktype_t type) {
fprintf(stderr,
isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_PRINTLOCK,
"rwlock %p thread %lu %s(%s): %s, %u active, "
"%u granted, %u rwaiting, %u wwaiting\n"),
rwl, isc_thread_self(), operation,
(type == isc_rwlocktype_read ?
isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_READ, "read") :
isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_WRITE, "write")),
(rwl->type == isc_rwlocktype_read ?
isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_READING, "reading") :
isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_WRITING, "writing")),
rwl->active, rwl->granted, rwl->readers_waiting,
rwl->writers_waiting);
}
#endif
isc_result_t
isc_rwlock_init(isc_rwlock_t *rwl, unsigned int read_quota,
unsigned int write_quota)
{
isc_result_t result;
REQUIRE(rwl != NULL);
rwl->magic = 0;
#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG)
rwl->write_requests = 0;
rwl->write_completions = 0;
rwl->cnt_and_flag = 0;
rwl->readers_waiting = 0;
rwl->write_granted = 0;
if (read_quota != 0) {
UNEXPECTED_ERROR(__FILE__, __LINE__,
"read quota is not supported");
}
if (write_quota == 0)
write_quota = RWLOCK_DEFAULT_WRITE_QUOTA;
rwl->write_quota = write_quota;
#else
rwl->type = isc_rwlocktype_read;
rwl->original = isc_rwlocktype_none;
rwl->active = 0;
rwl->granted = 0;
rwl->readers_waiting = 0;
rwl->writers_waiting = 0;
if (read_quota == 0)
read_quota = RWLOCK_DEFAULT_READ_QUOTA;
rwl->read_quota = read_quota;
if (write_quota == 0)
write_quota = RWLOCK_DEFAULT_WRITE_QUOTA;
rwl->write_quota = write_quota;
#endif
result = isc_mutex_init(&rwl->lock);
if (result != ISC_R_SUCCESS)
return (result);
result = isc_condition_init(&rwl->readable);
if (result != ISC_R_SUCCESS) {
UNEXPECTED_ERROR(__FILE__, __LINE__,
"isc_condition_init(readable) %s: %s",
isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,
ISC_MSG_FAILED, "failed"),
isc_result_totext(result));
result = ISC_R_UNEXPECTED;
goto destroy_lock;
}
result = isc_condition_init(&rwl->writeable);
if (result != ISC_R_SUCCESS) {
UNEXPECTED_ERROR(__FILE__, __LINE__,
"isc_condition_init(writeable) %s: %s",
isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,
ISC_MSG_FAILED, "failed"),
isc_result_totext(result));
result = ISC_R_UNEXPECTED;
goto destroy_rcond;
}
rwl->magic = RWLOCK_MAGIC;
return (ISC_R_SUCCESS);
destroy_rcond:
(void)isc_condition_destroy(&rwl->readable);
destroy_lock:
DESTROYLOCK(&rwl->lock);
return (result);
}
void
isc_rwlock_destroy(isc_rwlock_t *rwl) {
REQUIRE(VALID_RWLOCK(rwl));
#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG)
REQUIRE(rwl->write_requests == rwl->write_completions &&
rwl->cnt_and_flag == 0 && rwl->readers_waiting == 0);
#else
LOCK(&rwl->lock);
REQUIRE(rwl->active == 0 &&
rwl->readers_waiting == 0 &&
rwl->writers_waiting == 0);
UNLOCK(&rwl->lock);
#endif
rwl->magic = 0;
(void)isc_condition_destroy(&rwl->readable);
(void)isc_condition_destroy(&rwl->writeable);
DESTROYLOCK(&rwl->lock);
}
#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG)
#define WRITER_ACTIVE 0x1
#define READER_INCR 0x2
isc_result_t
isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
isc_int32_t cntflag;
REQUIRE(VALID_RWLOCK(rwl));
#ifdef ISC_RWLOCK_TRACE
print_lock(isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_PRELOCK, "prelock"), rwl, type);
#endif
if (type == isc_rwlocktype_read) {
if (rwl->write_requests != rwl->write_completions) {
LOCK(&rwl->lock);
if (rwl->write_requests != rwl->write_completions) {
rwl->readers_waiting++;
WAIT(&rwl->readable, &rwl->lock);
rwl->readers_waiting--;
}
UNLOCK(&rwl->lock);
}
cntflag = isc_atomic_xadd(&rwl->cnt_and_flag, READER_INCR);
while (1) {
if ((rwl->cnt_and_flag & WRITER_ACTIVE) == 0)
break;
LOCK(&rwl->lock);
rwl->readers_waiting++;
if ((rwl->cnt_and_flag & WRITER_ACTIVE) != 0)
WAIT(&rwl->readable, &rwl->lock);
rwl->readers_waiting--;
UNLOCK(&rwl->lock);
}
rwl->write_granted = 0;
} else {
isc_int32_t prev_writer;
prev_writer = isc_atomic_xadd(&rwl->write_requests, 1);
while (rwl->write_completions != prev_writer) {
LOCK(&rwl->lock);
if (rwl->write_completions != prev_writer) {
WAIT(&rwl->writeable, &rwl->lock);
UNLOCK(&rwl->lock);
continue;
}
UNLOCK(&rwl->lock);
break;
}
while (1) {
cntflag = isc_atomic_cmpxchg(&rwl->cnt_and_flag, 0,
WRITER_ACTIVE);
if (cntflag == 0)
break;
LOCK(&rwl->lock);
if (rwl->cnt_and_flag != 0)
WAIT(&rwl->writeable, &rwl->lock);
UNLOCK(&rwl->lock);
}
INSIST((rwl->cnt_and_flag & WRITER_ACTIVE) != 0);
rwl->write_granted++;
}
#ifdef ISC_RWLOCK_TRACE
print_lock(isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_POSTLOCK, "postlock"), rwl, type);
#endif
return (ISC_R_SUCCESS);
}
isc_result_t
isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
isc_int32_t cntflag;
REQUIRE(VALID_RWLOCK(rwl));
#ifdef ISC_RWLOCK_TRACE
print_lock(isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_PRELOCK, "prelock"), rwl, type);
#endif
if (type == isc_rwlocktype_read) {
if (rwl->write_requests != rwl->write_completions)
return (ISC_R_LOCKBUSY);
cntflag = isc_atomic_xadd(&rwl->cnt_and_flag, READER_INCR);
if ((cntflag & WRITER_ACTIVE) != 0) {
cntflag = isc_atomic_xadd(&rwl->cnt_and_flag,
-READER_INCR);
if (cntflag == READER_INCR &&
rwl->write_completions != rwl->write_requests) {
LOCK(&rwl->lock);
BROADCAST(&rwl->writeable);
UNLOCK(&rwl->lock);
}
return (ISC_R_LOCKBUSY);
}
} else {
cntflag = isc_atomic_cmpxchg(&rwl->cnt_and_flag, 0,
WRITER_ACTIVE);
if (cntflag != 0)
return (ISC_R_LOCKBUSY);
(void)isc_atomic_xadd(&rwl->write_completions, -1);
rwl->write_granted++;
}
#ifdef ISC_RWLOCK_TRACE
print_lock(isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_POSTLOCK, "postlock"), rwl, type);
#endif
return (ISC_R_SUCCESS);
}
isc_result_t
isc_rwlock_tryupgrade(isc_rwlock_t *rwl) {
isc_int32_t prevcnt;
REQUIRE(VALID_RWLOCK(rwl));
prevcnt = isc_atomic_cmpxchg(&rwl->cnt_and_flag,
READER_INCR, WRITER_ACTIVE);
INSIST((prevcnt & WRITER_ACTIVE) == 0 &&
(prevcnt & ~WRITER_ACTIVE) != 0);
if (prevcnt == READER_INCR) {
(void)isc_atomic_xadd(&rwl->write_completions, -1);
} else
return (ISC_R_LOCKBUSY);
return (ISC_R_SUCCESS);
}
void
isc_rwlock_downgrade(isc_rwlock_t *rwl) {
isc_int32_t prev_readers;
REQUIRE(VALID_RWLOCK(rwl));
prev_readers = isc_atomic_xadd(&rwl->cnt_and_flag, READER_INCR);
INSIST((prev_readers & WRITER_ACTIVE) != 0);
(void)isc_atomic_xadd(&rwl->cnt_and_flag, -WRITER_ACTIVE);
(void)isc_atomic_xadd(&rwl->write_completions, 1);
LOCK(&rwl->lock);
if (rwl->readers_waiting > 0)
BROADCAST(&rwl->readable);
UNLOCK(&rwl->lock);
}
isc_result_t
isc_rwlock_unlock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
isc_int32_t prev_cnt;
REQUIRE(VALID_RWLOCK(rwl));
#ifdef ISC_RWLOCK_TRACE
print_lock(isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_PREUNLOCK, "preunlock"), rwl, type);
#endif
if (type == isc_rwlocktype_read) {
prev_cnt = isc_atomic_xadd(&rwl->cnt_and_flag, -READER_INCR);
if (prev_cnt == READER_INCR &&
rwl->write_completions != rwl->write_requests) {
LOCK(&rwl->lock);
BROADCAST(&rwl->writeable);
UNLOCK(&rwl->lock);
}
} else {
isc_boolean_t wakeup_writers = ISC_TRUE;
(void)isc_atomic_xadd(&rwl->cnt_and_flag, -WRITER_ACTIVE);
(void)isc_atomic_xadd(&rwl->write_completions, 1);
if (rwl->write_granted >= rwl->write_quota ||
rwl->write_requests == rwl->write_completions ||
(rwl->cnt_and_flag & ~WRITER_ACTIVE) != 0) {
LOCK(&rwl->lock);
if (rwl->readers_waiting > 0) {
wakeup_writers = ISC_FALSE;
BROADCAST(&rwl->readable);
}
UNLOCK(&rwl->lock);
}
if (rwl->write_requests != rwl->write_completions &&
wakeup_writers) {
LOCK(&rwl->lock);
BROADCAST(&rwl->writeable);
UNLOCK(&rwl->lock);
}
}
#ifdef ISC_RWLOCK_TRACE
print_lock(isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_POSTUNLOCK, "postunlock"),
rwl, type);
#endif
return (ISC_R_SUCCESS);
}
#else
static isc_result_t
doit(isc_rwlock_t *rwl, isc_rwlocktype_t type, isc_boolean_t nonblock) {
isc_boolean_t skip = ISC_FALSE;
isc_boolean_t done = ISC_FALSE;
isc_result_t result = ISC_R_SUCCESS;
REQUIRE(VALID_RWLOCK(rwl));
LOCK(&rwl->lock);
#ifdef ISC_RWLOCK_TRACE
print_lock(isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_PRELOCK, "prelock"), rwl, type);
#endif
if (type == isc_rwlocktype_read) {
if (rwl->readers_waiting != 0)
skip = ISC_TRUE;
while (!done) {
if (!skip &&
((rwl->active == 0 ||
(rwl->type == isc_rwlocktype_read &&
(rwl->writers_waiting == 0 ||
rwl->granted < rwl->read_quota)))))
{
rwl->type = isc_rwlocktype_read;
rwl->active++;
rwl->granted++;
done = ISC_TRUE;
} else if (nonblock) {
result = ISC_R_LOCKBUSY;
done = ISC_TRUE;
} else {
skip = ISC_FALSE;
rwl->readers_waiting++;
WAIT(&rwl->readable, &rwl->lock);
rwl->readers_waiting--;
}
}
} else {
if (rwl->writers_waiting != 0)
skip = ISC_TRUE;
while (!done) {
if (!skip && rwl->active == 0) {
rwl->type = isc_rwlocktype_write;
rwl->active = 1;
rwl->granted++;
done = ISC_TRUE;
} else if (nonblock) {
result = ISC_R_LOCKBUSY;
done = ISC_TRUE;
} else {
skip = ISC_FALSE;
rwl->writers_waiting++;
WAIT(&rwl->writeable, &rwl->lock);
rwl->writers_waiting--;
}
}
}
#ifdef ISC_RWLOCK_TRACE
print_lock(isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_POSTLOCK, "postlock"), rwl, type);
#endif
UNLOCK(&rwl->lock);
return (result);
}
isc_result_t
isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
return (doit(rwl, type, ISC_FALSE));
}
isc_result_t
isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
return (doit(rwl, type, ISC_TRUE));
}
isc_result_t
isc_rwlock_tryupgrade(isc_rwlock_t *rwl) {
isc_result_t result = ISC_R_SUCCESS;
REQUIRE(VALID_RWLOCK(rwl));
LOCK(&rwl->lock);
REQUIRE(rwl->type == isc_rwlocktype_read);
REQUIRE(rwl->active != 0);
if (rwl->active == 1) {
rwl->original = (rwl->original == isc_rwlocktype_none) ?
isc_rwlocktype_read : isc_rwlocktype_none;
rwl->type = isc_rwlocktype_write;
} else
result = ISC_R_LOCKBUSY;
UNLOCK(&rwl->lock);
return (result);
}
void
isc_rwlock_downgrade(isc_rwlock_t *rwl) {
REQUIRE(VALID_RWLOCK(rwl));
LOCK(&rwl->lock);
REQUIRE(rwl->type == isc_rwlocktype_write);
REQUIRE(rwl->active == 1);
rwl->type = isc_rwlocktype_read;
rwl->original = (rwl->original == isc_rwlocktype_none) ?
isc_rwlocktype_write : isc_rwlocktype_none;
if (rwl->original == isc_rwlocktype_none &&
(rwl->writers_waiting == 0 || rwl->granted < rwl->read_quota) &&
rwl->readers_waiting > 0)
BROADCAST(&rwl->readable);
UNLOCK(&rwl->lock);
}
isc_result_t
isc_rwlock_unlock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
REQUIRE(VALID_RWLOCK(rwl));
LOCK(&rwl->lock);
REQUIRE(rwl->type == type);
UNUSED(type);
#ifdef ISC_RWLOCK_TRACE
print_lock(isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_PREUNLOCK, "preunlock"), rwl, type);
#endif
INSIST(rwl->active > 0);
rwl->active--;
if (rwl->active == 0) {
if (rwl->original != isc_rwlocktype_none) {
rwl->type = rwl->original;
rwl->original = isc_rwlocktype_none;
}
if (rwl->type == isc_rwlocktype_read) {
rwl->granted = 0;
if (rwl->writers_waiting > 0) {
rwl->type = isc_rwlocktype_write;
SIGNAL(&rwl->writeable);
} else if (rwl->readers_waiting > 0) {
BROADCAST(&rwl->readable);
}
} else {
if (rwl->readers_waiting > 0) {
if (rwl->writers_waiting > 0 &&
rwl->granted < rwl->write_quota) {
SIGNAL(&rwl->writeable);
} else {
rwl->granted = 0;
rwl->type = isc_rwlocktype_read;
BROADCAST(&rwl->readable);
}
} else if (rwl->writers_waiting > 0) {
rwl->granted = 0;
SIGNAL(&rwl->writeable);
} else {
rwl->granted = 0;
}
}
}
INSIST(rwl->original == isc_rwlocktype_none);
#ifdef ISC_RWLOCK_TRACE
print_lock(isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK,
ISC_MSG_POSTUNLOCK, "postunlock"),
rwl, type);
#endif
UNLOCK(&rwl->lock);
return (ISC_R_SUCCESS);
}
#endif
#else
isc_result_t
isc_rwlock_init(isc_rwlock_t *rwl, unsigned int read_quota,
unsigned int write_quota)
{
REQUIRE(rwl != NULL);
UNUSED(read_quota);
UNUSED(write_quota);
rwl->type = isc_rwlocktype_read;
rwl->active = 0;
rwl->magic = RWLOCK_MAGIC;
return (ISC_R_SUCCESS);
}
isc_result_t
isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
REQUIRE(VALID_RWLOCK(rwl));
if (type == isc_rwlocktype_read) {
if (rwl->type != isc_rwlocktype_read && rwl->active != 0)
return (ISC_R_LOCKBUSY);
rwl->type = isc_rwlocktype_read;
rwl->active++;
} else {
if (rwl->active != 0)
return (ISC_R_LOCKBUSY);
rwl->type = isc_rwlocktype_write;
rwl->active = 1;
}
return (ISC_R_SUCCESS);
}
isc_result_t
isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
return (isc_rwlock_lock(rwl, type));
}
isc_result_t
isc_rwlock_tryupgrade(isc_rwlock_t *rwl) {
isc_result_t result = ISC_R_SUCCESS;
REQUIRE(VALID_RWLOCK(rwl));
REQUIRE(rwl->type == isc_rwlocktype_read);
REQUIRE(rwl->active != 0);
if (rwl->active == 1)
rwl->type = isc_rwlocktype_write;
else
result = ISC_R_LOCKBUSY;
return (result);
}
void
isc_rwlock_downgrade(isc_rwlock_t *rwl) {
REQUIRE(VALID_RWLOCK(rwl));
REQUIRE(rwl->type == isc_rwlocktype_write);
REQUIRE(rwl->active == 1);
rwl->type = isc_rwlocktype_read;
}
isc_result_t
isc_rwlock_unlock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
REQUIRE(VALID_RWLOCK(rwl));
REQUIRE(rwl->type == type);
UNUSED(type);
INSIST(rwl->active > 0);
rwl->active--;
return (ISC_R_SUCCESS);
}
void
isc_rwlock_destroy(isc_rwlock_t *rwl) {
REQUIRE(rwl != NULL);
REQUIRE(rwl->active == 0);
rwl->magic = 0;
}
#endif