src/java.base/share/classes/jdk/internal/misc/X-ScopedMemoryAccess.java.template

/*
 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.  Oracle designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Oracle in the LICENSE file that accompanied this code.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */

package jdk.internal.misc;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.foreign.MemorySegment;
import java.lang.ref.Reference;
import java.io.FileDescriptor;
import java.util.function.Supplier;

import jdk.internal.access.foreign.MappedMemoryUtilsProxy;
import jdk.internal.access.JavaNioAccess;
import jdk.internal.access.SharedSecrets;
import jdk.internal.foreign.AbstractMemorySegmentImpl;
import jdk.internal.foreign.MemorySessionImpl;
import jdk.internal.util.ArraysSupport;
import jdk.internal.vm.annotation.ForceInline;
import jdk.internal.vm.vector.VectorSupport;


/**
 * This class defines low-level methods to access on-heap and off-heap memory. The methods in this class
 * can be thought of as thin wrappers around methods provided in the {@link Unsafe} class. All the methods in this
 * class accept one or more {@link MemorySessionImpl} parameter, which is used to validate as to whether access to memory
 * can be performed in a safe fashion - more specifically, to ensure that the memory being accessed has not
 * already been released (which would result in a hard VM crash).
 * <p>
 * Accessing and releasing memory from a single thread is not problematic - after all, a given thread cannot,
 * at the same time, access a memory region <em>and</em> free it. But ensuring correctness of memory access
 * when multiple threads are involved is much trickier, as there can be cases where a thread is accessing
 * a memory region while another thread is releasing it.
 * <p>
 * This class provides tools to manage races when multiple threads are accessing and/or releasing the same memory
 * session concurrently. More specifically, when a thread wants to release a memory session, it should call the
 * {@link ScopedMemoryAccess#closeScope(MemorySessionImpl)} method. This method initiates thread-local handshakes with all the other VM threads,
 * which are then stopped one by one. If any thread is found accessing a resource associated to the very memory session
 * being closed, the handshake fails, and the session will not be closed.
 * <p>
 * This synchronization strategy relies on the idea that accessing memory is atomic with respect to checking the
 * validity of the session associated with that memory region - that is, a thread that wants to perform memory access will be
 * suspended either <em>before</em> a liveness check or <em>after</em> the memory access. To ensure this atomicity,
 * all methods in this class are marked with the special {@link Scoped} annotation, which is recognized by the VM,
 * and used during the thread-local handshake to detect (and stop) threads performing potentially problematic memory access
 * operations. Additionally, to make sure that the session object(s) of the memory being accessed is always
 * reachable during an access operation, all the methods in this class add reachability fences around the underlying
 * unsafe access.
 * <p>
 * This form of synchronization allows APIs to use plain memory access without any other form of synchronization
 * which might be deemed to expensive; in other words, this approach prioritizes the performance of memory access over
 * that of releasing a shared memory resource.
 */
public final class ScopedMemoryAccess {

    private static final Unsafe UNSAFE = Unsafe.getUnsafe();

    private static native void registerNatives();
    static {
        registerNatives();
    }

    public void closeScope(MemorySessionImpl session, ScopedAccessError error) {
        closeScope0(session, error);
    }

    native void closeScope0(MemorySessionImpl session, ScopedAccessError error);

    private ScopedMemoryAccess() {}

    private static final ScopedMemoryAccess theScopedMemoryAccess = new ScopedMemoryAccess();

    public static ScopedMemoryAccess getScopedMemoryAccess() {
        return theScopedMemoryAccess;
    }

    public static final class ScopedAccessError extends Error {

        @SuppressWarnings("serial")
        private final Supplier<RuntimeException> runtimeExceptionSupplier;

        public ScopedAccessError(Supplier<RuntimeException> runtimeExceptionSupplier) {
            super("Invalid memory access", null, false, false);
            this.runtimeExceptionSupplier = runtimeExceptionSupplier;
        }

        static final long serialVersionUID = 1L;

        public final RuntimeException newRuntimeException() {
            return runtimeExceptionSupplier.get();
        }
    }

    @Target({ElementType.METHOD, ElementType.CONSTRUCTOR})
    @Retention(RetentionPolicy.RUNTIME)
    @interface Scoped { }

    // bulk ops

    @ForceInline
    public void copyMemory(MemorySessionImpl srcSession, MemorySessionImpl dstSession,
                                   Object srcBase, long srcOffset,
                                   Object destBase, long destOffset,
                                   long bytes) {
          try {
              copyMemoryInternal(srcSession, dstSession, srcBase, srcOffset, destBase, destOffset, bytes);
          } catch (ScopedAccessError ex) {
              throw ex.newRuntimeException();
          }
    }

    @ForceInline @Scoped
    private void copyMemoryInternal(MemorySessionImpl srcSession, MemorySessionImpl dstSession,
                               Object srcBase, long srcOffset,
                               Object destBase, long destOffset,
                               long bytes) {
        try {
            if (srcSession != null) {
                srcSession.checkValidStateRaw();
            }
            if (dstSession != null) {
                dstSession.checkValidStateRaw();
            }
            UNSAFE.copyMemory(srcBase, srcOffset, destBase, destOffset, bytes);
        } finally {
            Reference.reachabilityFence(srcSession);
            Reference.reachabilityFence(dstSession);
        }
    }

    @ForceInline
    public void copySwapMemory(MemorySessionImpl srcSession, MemorySessionImpl dstSession,
                                   Object srcBase, long srcOffset,
                                   Object destBase, long destOffset,
                                   long bytes, long elemSize) {
          try {
              copySwapMemoryInternal(srcSession, dstSession, srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
          } catch (ScopedAccessError ex) {
              throw ex.newRuntimeException();
          }
    }

    @ForceInline @Scoped
    private void copySwapMemoryInternal(MemorySessionImpl srcSession, MemorySessionImpl dstSession,
                               Object srcBase, long srcOffset,
                               Object destBase, long destOffset,
                               long bytes, long elemSize) {
        try {
            if (srcSession != null) {
                srcSession.checkValidStateRaw();
            }
            if (dstSession != null) {
                dstSession.checkValidStateRaw();
            }
            UNSAFE.copySwapMemory(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
        } finally {
            Reference.reachabilityFence(srcSession);
            Reference.reachabilityFence(dstSession);
        }
    }

    @ForceInline
    public void setMemory(MemorySessionImpl session, Object o, long offset, long bytes, byte value) {
        try {
            setMemoryInternal(session, o, offset, bytes, value);
        } catch (ScopedAccessError ex) {
            throw ex.newRuntimeException();
        }
    }

    @ForceInline @Scoped
    private void setMemoryInternal(MemorySessionImpl session, Object o, long offset, long bytes, byte value) {
        try {
            if (session != null) {
                session.checkValidStateRaw();
            }
            UNSAFE.setMemory(o, offset, bytes, value);
        } finally {
            Reference.reachabilityFence(session);
        }
    }

    @ForceInline
    public int vectorizedMismatch(MemorySessionImpl aSession, MemorySessionImpl bSession,
                                             Object a, long aOffset,
                                             Object b, long bOffset,
                                             int length,
                                             int log2ArrayIndexScale) {
        try {
            return vectorizedMismatchInternal(aSession, bSession, a, aOffset, b, bOffset, length, log2ArrayIndexScale);
        } catch (ScopedAccessError ex) {
            throw ex.newRuntimeException();
        }
    }

    @ForceInline @Scoped
    private int vectorizedMismatchInternal(MemorySessionImpl aSession, MemorySessionImpl bSession,
                                             Object a, long aOffset,
                                             Object b, long bOffset,
                                             int length,
                                             int log2ArrayIndexScale) {
        try {
            if (aSession != null) {
                aSession.checkValidStateRaw();
            }
            if (bSession != null) {
                bSession.checkValidStateRaw();
            }
            return ArraysSupport.vectorizedMismatch(a, aOffset, b, bOffset, length, log2ArrayIndexScale);
        } finally {
            Reference.reachabilityFence(aSession);
            Reference.reachabilityFence(bSession);
        }
    }

    @ForceInline
    public boolean isLoaded(MemorySessionImpl session, MappedMemoryUtilsProxy mappedUtils, long address, boolean isSync, long size) {
        try {
            return isLoadedInternal(session, mappedUtils, address, isSync, size);
        } catch (ScopedAccessError ex) {
            throw ex.newRuntimeException();
        }
    }

    @ForceInline @Scoped
    public boolean isLoadedInternal(MemorySessionImpl session, MappedMemoryUtilsProxy mappedUtils, long address, boolean isSync, long size) {
        try {
            if (session != null) {
                session.checkValidStateRaw();
            }
            return mappedUtils.isLoaded(address, isSync, size);
        } finally {
            Reference.reachabilityFence(session);
        }
    }

    @ForceInline
    public void load(MemorySessionImpl session, MappedMemoryUtilsProxy mappedUtils, long address, boolean isSync, long size) {
        try {
            loadInternal(session, mappedUtils, address, isSync, size);
        } catch (ScopedAccessError ex) {
            throw ex.newRuntimeException();
        }
    }

    @ForceInline @Scoped
    public void loadInternal(MemorySessionImpl session, MappedMemoryUtilsProxy mappedUtils, long address, boolean isSync, long size) {
        try {
            if (session != null) {
                session.checkValidStateRaw();
            }
            mappedUtils.load(address, isSync, size);
        } finally {
            Reference.reachabilityFence(session);
        }
    }

    @ForceInline
    public void unload(MemorySessionImpl session, MappedMemoryUtilsProxy mappedUtils, long address, boolean isSync, long size) {
        try {
            unloadInternal(session, mappedUtils, address, isSync, size);
        } catch (ScopedAccessError ex) {
            throw ex.newRuntimeException();
        }
    }

    @ForceInline @Scoped
    public void unloadInternal(MemorySessionImpl session, MappedMemoryUtilsProxy mappedUtils, long address, boolean isSync, long size) {
        try {
            if (session != null) {
                session.checkValidStateRaw();
            }
            mappedUtils.unload(address, isSync, size);
        } finally {
            Reference.reachabilityFence(session);
        }
    }

    @ForceInline
    public void force(MemorySessionImpl session, MappedMemoryUtilsProxy mappedUtils, FileDescriptor fd, long address, boolean isSync, long index, long length) {
        try {
            forceInternal(session, mappedUtils, fd, address, isSync, index, length);
        } catch (ScopedAccessError ex) {
            throw ex.newRuntimeException();
        }
    }

    @ForceInline @Scoped
    public void forceInternal(MemorySessionImpl session, MappedMemoryUtilsProxy mappedUtils, FileDescriptor fd, long address, boolean isSync, long index, long length) {
        try {
            if (session != null) {
                session.checkValidStateRaw();
            }
            mappedUtils.force(fd, address, isSync, index, length);
        } finally {
            Reference.reachabilityFence(session);
        }
    }

    // MemorySegment vector access ops

    @ForceInline
    public static
    <V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>>
    V loadFromMemorySegment(Class<? extends V> vmClass, Class<E> e, int length,
                         AbstractMemorySegmentImpl msp, long offset,
                         S s,
                         VectorSupport.LoadOperation<AbstractMemorySegmentImpl, V, S> defaultImpl) {

        try {
            return loadFromMemorySegmentScopedInternal(
                    msp.sessionImpl(),
                    vmClass, e, length,
                    msp, offset,
                    s,
                    defaultImpl);
        } catch (ScopedAccessError ex) {
            throw ex.newRuntimeException();
        }
    }

    @Scoped
    @ForceInline
    private static
    <V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>>
    V loadFromMemorySegmentScopedInternal(MemorySessionImpl session,
                                          Class<? extends V> vmClass, Class<E> e, int length,
                                          AbstractMemorySegmentImpl msp, long offset,
                                          S s,
                                          VectorSupport.LoadOperation<AbstractMemorySegmentImpl, V, S> defaultImpl) {
        try {
            session.checkValidStateRaw();

            return VectorSupport.load(vmClass, e, length,
                    msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, true,
                    msp, offset, s,
                    defaultImpl);
        } finally {
            Reference.reachabilityFence(session);
        }
    }

    @ForceInline
    public static
    <V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>,
     M extends VectorSupport.VectorMask<E>>
    V loadFromMemorySegmentMasked(Class<? extends V> vmClass, Class<M> maskClass, Class<E> e,
                                  int length, AbstractMemorySegmentImpl msp, long offset, M m, S s, int offsetInRange,
                                  VectorSupport.LoadVectorMaskedOperation<AbstractMemorySegmentImpl, V, S, M> defaultImpl) {

        try {
            return loadFromMemorySegmentMaskedScopedInternal(
                    msp.sessionImpl(),
                    vmClass, maskClass, e, length,
                    msp, offset, m,
                    s, offsetInRange,
                    defaultImpl);
        } catch (ScopedAccessError ex) {
            throw ex.newRuntimeException();
        }
    }

    @Scoped
    @ForceInline
    private static
    <V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>,
     M extends VectorSupport.VectorMask<E>>
    V loadFromMemorySegmentMaskedScopedInternal(MemorySessionImpl session, Class<? extends V> vmClass,
                                                Class<M> maskClass, Class<E> e, int length,
                                                AbstractMemorySegmentImpl msp, long offset, M m,
                                                S s, int offsetInRange,
                                                VectorSupport.LoadVectorMaskedOperation<AbstractMemorySegmentImpl, V, S, M> defaultImpl) {
        try {
            session.checkValidStateRaw();

            return VectorSupport.loadMasked(vmClass, maskClass, e, length,
                    msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, true, m, offsetInRange,
                    msp, offset, s,
                    defaultImpl);
        } finally {
            Reference.reachabilityFence(session);
        }
    }

    @ForceInline
    public static
    <V extends VectorSupport.Vector<E>, E>
    void storeIntoMemorySegment(Class<? extends V> vmClass, Class<E> e, int length,
                                V v,
                                AbstractMemorySegmentImpl msp, long offset,
                                VectorSupport.StoreVectorOperation<AbstractMemorySegmentImpl, V> defaultImpl) {

        try {
            storeIntoMemorySegmentScopedInternal(
                    msp.sessionImpl(),
                    vmClass, e, length,
                    v,
                    msp, offset,
                    defaultImpl);
        } catch (ScopedAccessError ex) {
            throw ex.newRuntimeException();
        }
    }

    @Scoped
    @ForceInline
    private static
    <V extends VectorSupport.Vector<E>, E>
    void storeIntoMemorySegmentScopedInternal(MemorySessionImpl session,
                                              Class<? extends V> vmClass, Class<E> e, int length,
                                              V v,
                                              AbstractMemorySegmentImpl msp, long offset,
                                              VectorSupport.StoreVectorOperation<AbstractMemorySegmentImpl, V> defaultImpl) {
        try {
            session.checkValidStateRaw();

            VectorSupport.store(vmClass, e, length,
                    msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, true,
                    v,
                    msp, offset,
                    defaultImpl);
        } finally {
            Reference.reachabilityFence(session);
        }
    }

    @ForceInline
    public static
    <V extends VectorSupport.Vector<E>, E, M extends VectorSupport.VectorMask<E>>
    void storeIntoMemorySegmentMasked(Class<? extends V> vmClass, Class<M> maskClass, Class<E> e,
                                      int length, V v, M m,
                                      AbstractMemorySegmentImpl msp, long offset,
                                      VectorSupport.StoreVectorMaskedOperation<AbstractMemorySegmentImpl, V, M> defaultImpl) {

        try {
            storeIntoMemorySegmentMaskedScopedInternal(
                    msp.sessionImpl(),
                    vmClass, maskClass, e, length,
                    v, m,
                    msp, offset,
                    defaultImpl);
        } catch (ScopedAccessError ex) {
            throw ex.newRuntimeException();
        }
    }

    @Scoped
    @ForceInline
    private static
    <V extends VectorSupport.Vector<E>, E, M extends VectorSupport.VectorMask<E>>
    void storeIntoMemorySegmentMaskedScopedInternal(MemorySessionImpl session,
                                                    Class<? extends V> vmClass, Class<M> maskClass,
                                                    Class<E> e, int length, V v, M m,
                                                    AbstractMemorySegmentImpl msp, long offset,
                                                    VectorSupport.StoreVectorMaskedOperation<AbstractMemorySegmentImpl, V, M> defaultImpl) {
        try {
            session.checkValidStateRaw();

            VectorSupport.storeMasked(vmClass, maskClass, e, length,
                    msp.unsafeGetBase(), msp.unsafeGetOffset() + offset, true,
                    v, m,
                    msp, offset,
                    defaultImpl);
        } finally {
            Reference.reachabilityFence(session);
        }
    }

    // typed-ops here

    // Note: all the accessor methods defined below take advantage of argument type profiling
    // (see src/hotspot/share/oops/methodData.cpp) which greatly enhances performance when the same accessor
    // method is used repeatedly with different 'base' objects.