Object3D.lookAt() in instanced meshes not as expected?

I'm building a particle system in which particles have directions they are pointing at and I want to reflect that in an instance mesh I use to present them, but something is not quite working... It seems that the center of rotation and/or the up axis are not quite right, and despite me console logging stuff I could not find the problem. Here's the code:

const Particles = (particleSystem: TparticleSystem) => {
  const mesh = useRef<THREE.InstancedMesh>(null);
  const light = useRef<THREE.PointLight>(null);

  const dummy = useMemo(() => new THREE.Object3D(), []);

  const pyramid = useMemo(() => {
    return {
      vertices: [1, 0, 0, 0, 0, 1, -1, 0, 0, 0, 0, -1, 0, 2, 0],
      indices: [0, 1, 2, 2, 3, 0, 0, 4, 1, 1, 4, 2, 2, 4, 3, 3, 4, 0],
    };
  }, []);

  useFrame(() => {
    if (particleSystem !== undefined) {
      particleSystem.update();
      particleSystem.move();
      particleSystem.particles.forEach((particle: Tparticle, index: number) => {
        dummy.position.set(particle.pos.x, particle.pos.y, particle.pos.z);

        dummy.lookAt(vec().copy(particle.pos).add(particle.dir)); // rotating weird

        dummy.scale.set(1, 2, 1);
        dummy.updateMatrix();
        // And apply the matrix to the instanced item
        if (mesh.current) mesh.current.setMatrixAt(index, dummy.matrix);
      });
      if (mesh.current) mesh.current.instanceMatrix.needsUpdate = true;
    }
  });

  return (
    <>
      <pointLight ref={light} distance={40} intensity={3} color="lightblue" />
      {particleSystem !== undefined && (
        <instancedMesh ref={mesh} args={[, , particleSystem.num]}>
          <polyhedronBufferGeometry
            args={[pyramid.vertices, pyramid.indices, 1, 0]}
          />
          <meshBasicMaterial
            color="#2596be"
            wireframe={Math.random() > 0.5 ? true : false}
          />
        </instancedMesh>
      )}
    </>
  );
};

I feel like maybe this is something to do with local and world coordinates, but I don't quite understand it.

Oh! and this vec stuff is just

const vec = function (x = 0, y = 0, z = 0) {
  return new THREE.Vector3(x, y, z);
};